Repository: mgholam/RaptorDB-Document Branch: master Commit: 4c1c2af5825e Files: 150 Total size: 1.2 MB Directory structure: gitextract_wcou_a2z/ ├── BuildVersion.cs ├── LICENSE ├── README.md ├── RaptorDB/ │ ├── AssemblyInfo.cs │ ├── DataTypes/ │ │ └── DataTypes.cs │ ├── Global.cs │ ├── Helper/ │ │ ├── Container.cs │ │ ├── MGRB.cs │ │ └── WAHBitarray2.cs │ ├── Indexes/ │ │ ├── BitmapIndex.cs │ │ ├── Cache.cs │ │ ├── Hoot.cs │ │ ├── IIndex.cs │ │ ├── ITokenizer.cs │ │ ├── IndexFile.cs │ │ ├── Indexes.cs │ │ ├── MGIndex.cs │ │ └── tokenizer.cs │ ├── Properties/ │ │ ├── Resources.Designer.cs │ │ └── Resources.resx │ ├── REST/ │ │ ├── aWebServer.cs │ │ └── rdbRest.cs │ ├── RaptorDB.cs │ ├── RaptorDB.csproj │ ├── RaptorDBServer.cs │ ├── Replication/ │ │ ├── Configuration.cs │ │ ├── Packets.cs │ │ ├── Readme.txt │ │ ├── ReplicationClient.cs │ │ ├── ReplicationServer.cs │ │ └── msg.txt │ ├── Storage/ │ │ ├── KeyStore.cs │ │ ├── KeyStoreHF.cs │ │ ├── StorageFile.cs │ │ ├── StorageFileHF.cs │ │ └── StringHF.cs │ ├── Views/ │ │ ├── Dynamic.cs │ │ ├── LINQQuery.cs │ │ ├── TaskQueue.cs │ │ ├── ViewHandler.cs │ │ ├── ViewManager.cs │ │ └── apimapper.cs │ ├── WEB/ │ │ ├── bundle.css │ │ ├── bundle.js │ │ ├── global.css │ │ └── index.html │ └── cron/ │ ├── CronDaemon.cs │ ├── CronJob.cs │ └── CronSchedule.cs ├── RaptorDB.Common/ │ ├── DataTypes.cs │ ├── FieldDescriptor.cs │ ├── IRaptorDB.cs │ ├── Interfaces.cs │ ├── LINQString.cs │ ├── Logger.cs │ ├── MiniLZO.cs │ ├── MurMurHash2.cs │ ├── NetworkClient.cs │ ├── Packets.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── RaptorDB.Common.csproj │ ├── RaptorDBClient.cs │ ├── SafeDictionary.cs │ ├── View.cs │ ├── ZipStorer.cs │ ├── fastBinaryJSON/ │ │ ├── BJSON.cs │ │ ├── BJsonParser.cs │ │ ├── BJsonSerializer.cs │ │ ├── Helper.cs │ │ └── dynamic.cs │ └── fastJSON/ │ ├── Formatter.cs │ ├── Getters.cs │ ├── Helper.cs │ ├── JSON.cs │ ├── JsonParser.cs │ ├── JsonSerializer.cs │ ├── Reflection.cs │ └── dynamic.cs ├── RaptorDBCore/ │ ├── RaptorDB/ │ │ └── RaptorDB.csproj │ ├── RaptorDb.Common/ │ │ └── RaptorDb.Common.csproj │ └── test/ │ ├── sample.cs │ └── test.csproj ├── RaptorDBServer/ │ ├── Installer.cs │ ├── Program.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── RaptorDBServer.csproj │ ├── Service1.Designer.cs │ ├── Service1.cs │ └── Service1.resx ├── RaptorDBTest.sln ├── RaptorDB_Doc.nuspec ├── RaptorDbCore.sln ├── Tools/ │ └── buildversion.ncs ├── Views/ │ ├── Class1.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── ServerSide.cs │ └── Views.csproj ├── WebStudio/ │ ├── README.md │ ├── build.cmd │ ├── deploy.cmd │ ├── package.json │ ├── rollup.config.js │ └── src/ │ ├── App.svelte │ ├── UI/ │ │ ├── Button.svelte │ │ ├── Modal.svelte │ │ ├── datatable.svelte │ │ └── nav.svelte │ ├── debug.js │ ├── global.css │ ├── index.html │ ├── main.js │ └── pages/ │ ├── dochistory.svelte │ ├── docsearch.svelte │ ├── docview.svelte │ ├── help.svelte │ ├── hfbrowser.svelte │ ├── query.svelte │ ├── schema.svelte │ ├── sysconfig.svelte │ └── sysinfo.svelte ├── build.cmd ├── datagridbinding/ │ ├── Form1.resources │ ├── Program.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── datagridbinding.csproj │ ├── frmMain.Designer.cs │ ├── frmMain.cs │ ├── frmMain.resx │ ├── frmStartup.Designer.cs │ ├── frmStartup.cs │ └── frmStartup.resx ├── history.txt ├── raptordb.snk ├── test script/ │ ├── run.cmd │ └── sample.cs ├── testing/ │ ├── AssemblyInfo.cs │ ├── Class1.cs │ ├── program.cs │ └── tests.csproj ├── testing.view └── vbTestConsole/ ├── App.config ├── Module1.vb ├── My Project/ │ ├── Application.Designer.vb │ ├── Application.myapp │ ├── AssemblyInfo.vb │ ├── Resources.Designer.vb │ ├── Resources.resx │ ├── Settings.Designer.vb │ └── Settings.settings └── vbtestconsole.vbproj ================================================ FILE CONTENTS ================================================ ================================================ FILE: BuildVersion.cs ================================================ using System.Reflection; // build number = 606 // build version = 4.0.10 [assembly: AssemblyVersion("4.0.0.0")] [assembly: AssemblyFileVersion("4.0.10.606")] ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2014 Mehdi Gholam Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # RaptorDB Document Store NoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication) see the article here : [http://www.codeproject.com/Articles/375413/RaptorDB-the-Document-Store] (http://www.codeproject.com/Articles/375413/RaptorDB-the-Document-Store) ## Quick Start First compile the source, then you can easily run any c# file like this: ``` # run any cs file c:\rdb\test script> ..\tools\nscript.exe sample.cs # or just run the batch file c:\rdb\test script> run.cmd ``` The `sample.cs` file now contains a comment section at the top for specifing references used which will tell `nscript.exe` where to find the dll files: ``` // ref : ..\output\raptordb.dll // ref : ..\output\raptordb.common.dll // ref : ..\faker.dll using System; using System.Collections.Generic; ... ``` ================================================ FILE: RaptorDB/AssemblyInfo.cs ================================================ using System.Reflection; using System.Security; [assembly: AssemblyTitle("RaptorDB Document Store")] [assembly: AssemblyDescription("NoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication)")] [assembly: AssemblyProduct("RaptorDB Document Store")] ================================================ FILE: RaptorDB/DataTypes/DataTypes.cs ================================================ using System; using RaptorDB.Common; namespace RaptorDB { /// /// Used to track ViewDelete usage for view rebuilds /// internal class View_delete { public Guid ID = Guid.NewGuid(); public string Viewname; public string Filter; } internal class View_insert { public Guid ID = Guid.NewGuid(); public string Viewname; public object RowObject; } internal class FullTextString { } internal class NoIndexing { } public interface IRowFiller { object FillRow(object row, object[] data); } internal interface IGetBytes { byte[] GetBytes(T obj); T GetObject(byte[] buffer, int offset, int count); } internal class RDBDataType { public static IGetBytes ByteHandler() { Type type = typeof(T); if (type == typeof(int)) return (IGetBytes)new int_handler(); else if (type == typeof(uint)) return (IGetBytes)new uint_handler(); else if (type == typeof(long)) return (IGetBytes)new long_handler(); else if (type == typeof(Guid)) return (IGetBytes)new guid_handler(); else if (type == typeof(string)) return (IGetBytes)new string_handler(); else if (type == typeof(DateTime)) return (IGetBytes)new datetime_handler(); else if (type == typeof(decimal)) return (IGetBytes)new decimal_handler(); else if (type == typeof(short)) return (IGetBytes)new short_handler(); else if (type == typeof(ushort)) return (IGetBytes)new ushort_handler(); else if (type == typeof(float)) return (IGetBytes)new float_handler(); else if (type == typeof(byte)) return (IGetBytes)new byte_handler(); else if (type == typeof(double)) return (IGetBytes)new double_handler(); return null; } public static byte GetByteSize(byte keysize) { byte size = 4; Type t = typeof(T); if (t == typeof(int)) size = 4; if (t == typeof(uint)) size = 4; if (t == typeof(long)) size = 8; if (t == typeof(Guid)) size = 16; if (t == typeof(DateTime)) size = 8; if (t == typeof(decimal)) size = 16; if (t == typeof(float)) size = 4; if (t == typeof(short)) size = 2; if (t == typeof(string)) size = keysize; if (t == typeof(byte)) size = 1; if (t == typeof(double)) size = 8; return size; } internal static object GetEmpty() { Type t = typeof(T); if (t == typeof(string)) return ""; return default(T); } } #region [ handlers ] internal class double_handler : IGetBytes { public byte[] GetBytes(double obj) { return BitConverter.GetBytes(obj); } public double GetObject(byte[] buffer, int offset, int count) { return BitConverter.ToDouble(buffer, offset); } } internal class byte_handler : IGetBytes { public byte[] GetBytes(byte obj) { return new byte[1] { obj }; } public byte GetObject(byte[] buffer, int offset, int count) { return buffer[offset]; } } internal class float_handler : IGetBytes { public byte[] GetBytes(float obj) { return BitConverter.GetBytes(obj); } public float GetObject(byte[] buffer, int offset, int count) { return BitConverter.ToSingle(buffer, offset); } } internal class decimal_handler : IGetBytes { public byte[] GetBytes(decimal obj) { byte[] b = new byte[16]; var bb = decimal.GetBits(obj); int index = 0; foreach (var d in bb) { byte[] db = Helper.GetBytes(d, false); Buffer.BlockCopy(db, 0, b, index, 4); index += 4; } return b; } public decimal GetObject(byte[] buffer, int offset, int count) { int[] i = new int[4]; i[0] = Helper.ToInt32(buffer, offset); offset += 4; i[1] = Helper.ToInt32(buffer, offset); offset += 4; i[2] = Helper.ToInt32(buffer, offset); offset += 4; i[3] = Helper.ToInt32(buffer, offset); offset += 4; return new decimal(i); } } internal class ushort_handler : IGetBytes { public byte[] GetBytes(ushort obj) { return Helper.GetBytes(obj, false); } public ushort GetObject(byte[] buffer, int offset, int count) { return (ushort)Helper.ToInt16(buffer, offset); } } internal class short_handler : IGetBytes { public byte[] GetBytes(short obj) { return Helper.GetBytes(obj, false); } public short GetObject(byte[] buffer, int offset, int count) { return Helper.ToInt16(buffer, offset); } } internal class string_handler : IGetBytes { public byte[] GetBytes(string obj) { return Helper.GetBytes(obj); } public string GetObject(byte[] buffer, int offset, int count) { return Helper.GetString(buffer, offset, (short)count); } } internal class int_handler : IGetBytes { public byte[] GetBytes(int obj) { return Helper.GetBytes(obj, false); } public int GetObject(byte[] buffer, int offset, int count) { return Helper.ToInt32(buffer, offset); } } internal class uint_handler : IGetBytes { public byte[] GetBytes(uint obj) { return Helper.GetBytes(obj, false); } public uint GetObject(byte[] buffer, int offset, int count) { return (uint)Helper.ToInt32(buffer, offset); } } internal class long_handler : IGetBytes { public byte[] GetBytes(long obj) { return Helper.GetBytes(obj, false); } public long GetObject(byte[] buffer, int offset, int count) { return Helper.ToInt64(buffer, offset); } } internal class guid_handler : IGetBytes { public byte[] GetBytes(Guid obj) { return obj.ToByteArray(); } public Guid GetObject(byte[] buffer, int offset, int count) { byte[] b = new byte[16]; Buffer.BlockCopy(buffer, offset, b, 0, 16); return new Guid(b); } } internal class datetime_handler : IGetBytes { public byte[] GetBytes(DateTime obj) { return Helper.GetBytes(obj.Ticks, false); } public DateTime GetObject(byte[] buffer, int offset, int count) { long ticks = Helper.ToInt64(buffer, offset); return new DateTime(ticks); } } #endregion } ================================================ FILE: RaptorDB/Global.cs ================================================ namespace RaptorDB { public class Global { /// /// Store bitmap as int offsets then switch over to bitarray /// public static int BitmapOffsetSwitchOverCount = 10; /// /// True = Save to other views in process , False = background save to other views /// public static bool BackgroundSaveToOtherViews = true; /// /// Default maximum string key size for indexes /// public static byte DefaultStringKeySize = 60; /// /// Free bitmap index memory on save /// public static bool FreeBitmapMemoryOnSave = false; /// /// Number of items in each index page (default = 10000) [Expert only, do not change] /// public static ushort PageItemCount = 10000; /// /// KeyStore save to disk timer /// public static int SaveIndexToDiskTimerSeconds = 1800; /// /// Flush the StorageFile stream immediately /// public static bool FlushStorageFileImmediately = false; /// /// Save doc as binary json /// public static bool SaveAsBinaryJSON = true; /// /// Remove completed tasks timer /// public static int TaskCleanupTimerSeconds = 3; /// /// Save to other views timer seconds if enabled /// public static int BackgroundSaveViewTimer = 1; /// /// How many items to process in a background view save event /// public static int BackgroundViewSaveBatchSize = 1000000; ///// ///// Check the restore folder for new backup files to restore ///// //public static int RestoreTimerSeconds = 10; // TODO : implement this /// /// Timer for full text indexing of original documents (default = 15 sec) /// public static int FullTextTimerSeconds = 15; /// /// How many documents to full text index in a batch /// public static int BackgroundFullTextIndexBatchSize = 10000; /// /// Free memory checking timer (default = 300 sec ~ 5 min) /// public static int FreeMemoryTimerSeconds = 5 * 60;// 1800; /// /// Memory usage limit for internal caching (default = 100 Mb) [using GC.GetTotalMemory()] /// public static long MemoryLimit = 100; /// /// Backup cron schedule (default = "0 * * * *" [every hour]) /// public static string BackupCronSchedule = "0 * * * *"; /// /// Require primary view to be defined for save, false = key/value store (default = true) /// public static bool RequirePrimaryView = true; /// /// Maximum documents in each package for replication /// public static int PackageSizeItemCountLimit = 10000; /// /// Process inbox timer (default = 60 sec) /// public static int ProcessInboxTimerSeconds = 60; /// /// Split the data storage files in MegaBytes (default 0 = off) [500 = 500mb] /// - You can set and unset this value anytime and it will operate from that point on. /// - If you unset (0) the value previous split files will remain and all the data will go to the last file. /// public static ushort SplitStorageFilesMegaBytes = 0; /// /// Compress the documents in the storage file if it is over this size (default = 100 Kilobytes) /// - You will be trading CPU for disk IO /// public static ushort CompressDocumentOverKiloBytes = 100; /// /// Disk block size for high frequency KV storage file (default = 2048) /// * Do not use anything under 512 with large string keys /// public static ushort HighFrequencyKVDiskBlockSize = 2048; /// /// String key MGIndex that stores keys in an external file for smaller index files /// public static bool EnableOptimizedStringIndex = true; /// /// Enable the Web Studio interface /// public static bool EnableWebStudio = false; /// /// Web Studio port (default = 91) /// public static short WebStudioPort = 91; /// /// Local machine access only Web Studio - no network access (default = true) /// public static bool LocalOnlyWebStudio = true; /// /// If True -> less memory use SafeSortedList and slower /// False -> more memory use SafeDictionary and faster /// public static bool UseLessMemoryStructures = false; //public static bool useSortedList = false; public static bool CompressBitmapBytes = false; public static bool SkipDocsOnViewInsert = false; } } ================================================ FILE: RaptorDB/Helper/Container.cs ================================================ using System.Collections.Generic; namespace RaptorDB { //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- class BitmapContainer : Container { public BitmapContainer() { } public BitmapContainer(int OneCount) { if (OneCount > Container.BSize) throw new System.Exception("OneCount > 65536"); else if (OneCount == Container.BSize) ALLONE = true; else { List l = new List(); while(OneCount>0) { if (OneCount > 64) l.Add(ulong.MaxValue); else l.Add(ulong.MaxValue< 0) return _onecount; lock (_lock) { long c = 0; foreach (var l in _values) c += BitCount(l); _onecount = c; return c; } } public override long CountZeros() { if (ALLONE) return 0; return BSize - CountOnes(); } public override IEnumerable GetBitIndexes() { lock (_lock) { if (ALLONE) { for (int i = 0; i < BSize; i++) yield return (ushort)i; } ushort c = 0; foreach (var l in _values) { for (int i = 0; i < 64; i++) { ulong mask = (ulong)1 << (63 - i); // high order bit get if ((l & mask) != 0) yield return (ushort)(c + i); } c += 64; } } } public override bool Get(long offset) { lock (_lock) { if (ALLONE) return true; int pos = (ushort)offset >> 6; int off = (int)(offset % 64); if (pos >= _values.Length) // out of range { return false; } ulong mask = (ulong)1 << (63 - off); // high order bit get return (_values[pos] & mask) != 0; } } public override void Set(long offset, bool val) { lock (_lock) { if (ALLONE) { if (val == true) return; // change to bits ALLONE = false; _values = new ulong[1024]; for (int i = 0; i < 1024; i++) _values[i] = ulong.MaxValue; } int pos = (ushort)offset >> 6; int off = (int)(offset % 64); _onecount = -1; if (_values == null) { _values = new ulong[0]; } if (pos >= _values.Length) // out of range { // resize var a = new ulong[pos + 1]; _values.CopyTo(a, 0); _values = a; } ulong mask = (ulong)1 << (63 - off); // high order bit get if (val) _values[pos] |= mask; else _values[pos] &= ~mask; Size = _values.Length * 64; } } public override bool ChangeRequired() { if (ALLONE) return false; if (CountOnes() == BSize) // -> all ones container return true; if (CountZeros() < CHGOVER) // -> inverted container return true; var offbytes = CountOnes() << 1; //*2 var bytes = _values.Length << 3; //*8 if (bytes > offbytes) return true; // -> offset container return false; } public override Container ToBitmap() { return Copy(); } public override Container Change() { if (ALLONE) return new BitmapContainer(true); if (CountOnes() == BSize) return new BitmapContainer(true); // create inverted if (CountZeros() < CHGOVER) return new InvertedContainer(Not().GetBitIndexes()); Container c = null; //if (Global.useSortedList) // c = new OffsetContainerSL(); //else c = new OffsetContainer(); foreach (var i in GetBitIndexes()) c.Set(i, true); return c; } public override Container Copy() { if (ALLONE) return new BitmapContainer(true); if (_values != null && _values.Length > 0) return new BitmapContainer(Values()); else return new BitmapContainer(); } public override Container Not() { lock (_lock) { if (ALLONE) return new BitmapContainer(); var vals = new ulong[1024]; // TODO : upto Size ?? for (int i = 0; i < 1024; i++) vals[i] = ulong.MaxValue; for (int i = 0; i < _values.Length; i++) vals[i] = ~_values[i]; return new BitmapContainer(vals); } } } //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- class OffsetContainer : Container { public OffsetContainer() { } public OffsetContainer(IEnumerable vals) { _values = new List(vals); Size = _values[_values.Count - 1]; } List _values = new List(); private object _lock = new object(); public ushort[] Values() { lock (_lock) return _values.ToArray(); } public override Container ToBitmap() { lock (_lock) { var c = new BitmapContainer(); foreach (var i in _values) c.Set(i, true); return c; } } public override Container Change() { if (CountOnes() == Container.BSize) return new BitmapContainer(true); if (CountZeros() < CHGOVER) return new InvertedContainer(ToBitmap().Not().GetBitIndexes()); return ToBitmap(); } public override bool ChangeRequired() { if (_values.Count > CHGOVER) return true; if (CountZeros() < CHGOVER) return true; return false; } public override Container Copy() { lock (_lock) { if (_values != null && _values.Count > 0) return new OffsetContainer(_values.ToArray()); else return new OffsetContainer(); } } public override long CountOnes() { return _values.Count; } public override long CountZeros() { return BSize - CountOnes(); } public override bool Get(long offset) { lock (_lock) { var i = _values.BinarySearch((ushort)offset); if (i >= 0) return true; return false; } } public override IEnumerable GetBitIndexes() { lock (_lock) foreach (var i in _values) yield return i; } public override void Set(long offset, bool val) { lock (_lock) { var i = _values.BinarySearch((ushort)offset); if (val == true) { if (i < 0) // not in array -> add { var c = ~i; if (c < _values.Count) _values.Insert(c, (ushort)offset); else _values.Add((ushort)offset); } } else if (i >= 0) { // remove from array _values.RemoveAt(i); } if (_values.Count > 0) Size = _values[_values.Count - 1]; else Size = -1; } } public override Container Not() { return ToBitmap().Not(); } } //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- class InvertedContainer : Container { public InvertedContainer() { } // list of zeros public InvertedContainer(IEnumerable vals) { _values = new List(vals); if (_values.Count > 0) Size = _values[_values.Count - 1]; else Size = Container.BSize; } List _values = new List(); private object _lock = new object(); public override Container ToBitmap() { lock (_lock) { var b = new BitmapContainer(); foreach (var i in GetBitIndexes()) { b.Set(i, true); } return b; } } public override Container Change() { if (CountZeros() > CHGOVER) return ToBitmap(); return Copy(); } public override bool ChangeRequired() { if (CountZeros() > CHGOVER) return true; return false; } public override Container Copy() { lock (_lock) { if (_values != null && _values.Count > 0) return new InvertedContainer(_values.ToArray()); else return new InvertedContainer(); } } public override long CountOnes() { return BSize - CountZeros(); } public override long CountZeros() { return _values.Count; } public override bool Get(long offset) { lock (_lock) { var i = _values.BinarySearch((ushort)offset); if (i >= 0) // in the list of zeros return false; return true; } } public override IEnumerable GetBitIndexes() { lock (_lock) { for (int i = 0; i < BSize; i++) { var j = _values.BinarySearch((ushort)i); if (j < 0) // not in the list of zeros yield return (ushort)i; } } } public override Container Not() { return ToBitmap().Not(); } public override void Set(long offset, bool val) { lock (_lock) { var i = _values.BinarySearch((ushort)offset); if (val == false) { if (i < 0) // not in array -> add { var c = ~i; if (c < _values.Count) _values.Insert(c, (ushort)offset); else _values.Add((ushort)offset); } } else if (i >= 0) { // remove from array _values.RemoveAt(i); } // fix : return correct size if (_values.Count > 0) Size = _values[_values.Count - 1]; else Size = -1; } } public ushort[] Values() { lock (_lock) return _values.ToArray(); } } //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------- public abstract class Container { internal const int BSize = 65536; internal const int CHGOVER = 4096; public abstract void Set(long offset, bool val); public abstract bool Get(long offset); public abstract long CountOnes(); public abstract long CountZeros(); public abstract IEnumerable GetBitIndexes(); public abstract bool ChangeRequired(); public abstract Container Change(); public abstract Container ToBitmap(); public abstract Container Copy(); public abstract Container Not(); public int Size = -1; //[MethodImpl(MethodImplOptions.AggressiveInlining)] public static int BitCount(ulong x) { x -= (x >> 1) & 0x5555555555555555UL; //put count of each 2 bits into those 2 bits x = (x & 0x3333333333333333UL) + ((x >> 2) & 0x3333333333333333UL); //put count of each 4 bits into those 4 bits x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0FUL; //put count of each 8 bits into those 8 bits return (int)((x * 0x0101010101010101UL) >> 56); //returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24) + ... } } public enum CTYPE { ALLONES , BITMAP , OFFSET , INV //,OFFSETSL } public class CData { public ushort i; public CTYPE t; public byte[] d; } public class MGRBData { public List c = new List(); } } ================================================ FILE: RaptorDB/Helper/MGRB.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; namespace RaptorDB { public class MGRB { public MGRB() { } internal MGRB(SafeSortedList containers) : this(containers, -1) { } internal MGRB(SafeSortedList containers, long size) { _containers = containers; var k = _containers.Keys(); _size = size; if (size <= 0)//== -1) { _size = 0; var l = k.Length - 1; if (l >= 0) _size = (k[l] << 16) + _containers.GetValue(l).Size; } } private SafeSortedList _containers = new SafeSortedList(); private long _size; private ushort _MASK = 0xffff; private object _lock = new object(); public bool isDirty = false; public long Length { get { return _size; } } public void Set(long position, bool val) { lock (_lock) { isDirty = true; if (_size < position && val == true) _size = position; var idx = (int)(position >> 16); Container c = null; if (_containers.TryGetValue(idx, out c) == false) { //if (Global.useSortedList) // c = new OffsetContainerSL(); //else c = new OffsetContainer(); // add container _containers.Add(idx, c); } c.Set(position & _MASK, val); //if (c.ChangeRequired()) // _containers[idx] = c.Change(); } } public bool Get(long position) { lock (_lock) { var idx = (int)(position >> 16); if (_containers.TryGetValue(idx, out Container c)) return c.Get(position & _MASK); return false; } } public MGRB And(MGRB B) { var v = new SafeSortedList(); var len = _size; if (B.Length < len) len = B.Length; var a = LastContainerIdx(); var b = B.LastContainerIdx(); var min = a; if (b < min) min = b; min++; for (int i = 0; i < min; i++) { Container ca = null; Container cb = null; _containers.TryGetValue(i, out ca); B._containers.TryGetValue(i, out cb); if (ca != null && cb != null) v.Add(i, containerAND(ca, cb)); } return new MGRB(v, len); } public MGRB Or(MGRB B) { var v = new SafeSortedList(); var len = _size; if (B.Length > len) len = B.Length; var a = LastContainerIdx(); var b = B.LastContainerIdx(); var max = a; if (b > max) max = b; max++; for (int i = 0; i < max; i++) { Container ca = null; Container cb = null; _containers.TryGetValue(i, out ca); B._containers.TryGetValue(i, out cb); if (ca == null && cb != null) v.Add(i, cb.Copy()); else if (cb == null && ca != null) v.Add(i, ca.Copy()); else if (ca != null && cb != null) v.Add(i, containerOR(ca, cb)); } return new MGRB(v, len); } public MGRB AndNot(MGRB b) { long c = _size; if (b._size > c) c = b._size; return And(b.Not(c)); } public MGRB Not() { var con = new SafeSortedList(); foreach (var c in _containers) { con.Add(c.Key, c.Value.Not()); } return new MGRB(con, _size); } public MGRB Not(long count) { var con = new SafeSortedList(); var c = count >> 16; for (int i = 0; i <= c; i++) { Container a = null; _containers.TryGetValue(i, out a); if (a == null) con.Add(i, new BitmapContainer(true)); else con.Add(i, a.Not()); } return new MGRB(con, count); } public static MGRB Fill(long count) { if (count == 0) return new MGRB(); var con = new SafeSortedList(); int i = 0; long c = count; while (count > 0) { if (count > Container.BSize) con.Add(i, new BitmapContainer(true)); else con.Add(i, new BitmapContainer((int)count)); count -= Container.BSize; i++; } return new MGRB(con, c); } public long CountOnes() { long c = 0; if (_size > 0) foreach (var i in _containers) c += i.Value.CountOnes(); return c; } public long CountZeros() { var c = CountOnes(); return _size - c; } public IEnumerable GetBitIndexes() { foreach (var c in _containers) { int i = c.Key << 16; foreach (var j in c.Value.GetBitIndexes()) yield return i + j; } } public MGRB Optimize() { lock (_lock) { var keys = _containers.Keys(); var remove = new List(); for (int i = 0; i < keys.Length; i++) { var k = keys[i]; var c = _containers[k]; if (c.CountOnes() == 0) remove.Add(k); //else if (c.CountZeros() < Container.CHGOVER) //{ // _containers[k] = new ZeroContainer(); //} else if (c.ChangeRequired()) _containers[k] = c.Change(); } foreach (var k in remove) _containers.Remove(k); return this; } } public MGRBData Serialize() { var d = new MGRBData(); foreach (var c in _containers) { var cd = new CData(); { cd.i = (ushort)c.Key; if (c.Value is BitmapContainer) { var bm = c.Value as BitmapContainer; cd.t = CTYPE.BITMAP; if (bm.ALLONE) cd.t = CTYPE.ALLONES; else { // get data cd.d = ToByteArray(bm.Values()); } } else if (c.Value is OffsetContainer) { var of = c.Value as OffsetContainer; cd.t = CTYPE.OFFSET; cd.d = ToByteArray(of.Values()); } else if (c.Value is InvertedContainer) { var inv = c.Value as InvertedContainer; cd.t = CTYPE.INV; cd.d = ToByteArray(inv.Values()); } //else //{ // var of = c.Value as OffsetContainerSL; // cd.t = CTYPE.OFFSETSL; // var b = new byte[cd.d.Length]; // int k = 0; // foreach (var i in of._values) // { // Buffer.BlockCopy(GetBytes(i.Key, false), 0, b, k, 2); // k += 2; // } // cd.d = b; //} d.c.Add(cd); } } return d; } public void Deserialize(MGRBData input) { foreach (var c in input.c) { Container con = null; if (c.t == CTYPE.ALLONES) { con = new BitmapContainer(true); } else if (c.t == CTYPE.BITMAP) { List list = new List(); var dataLen = c.d.Length; for (int i = 0; i < dataLen; i += 8) { list.Add(ToULong(c.d, i)); } con = new BitmapContainer(list.ToArray()); } else if (c.t == CTYPE.OFFSET) { List list = new List(); var dataLen = c.d.Length; for (int i = 0; i < dataLen; i += 2) { list.Add(ToUShort(c.d, i)); } con = new OffsetContainer(list); } else if (c.t == CTYPE.INV) { List list = new List(); var dataLen = c.d.Length; for (int i = 0; i < dataLen; i += 2) { list.Add(ToUShort(c.d, i)); } con = new InvertedContainer(list); } //else //{ // List list = new List(); // var dataLen = c.d.Length; // for (int i = 0; i < dataLen; i += 2) // { // list.Add(ToUShort(c.d, i)); // } // con = new OffsetContainerSL(list); //} _containers.Add(c.i, con); } var k = _containers.Keys(); var l = k.Length - 1; if (l >= 0) _size = (k[l] << 16) + _containers.GetValue(l).Size; } public MGRB Copy() { if (_containers.Count() > 0) { var o = Serialize(); var m = new MGRB(); m.Deserialize(o); return m; } return new MGRB(); } public int GetFirst() { int j = 0; foreach (var i in GetBitIndexes()) { j = i; break; } return j; } private int LastContainerIdx() { if (_containers.Count() > 0) return _containers.Keys()[_containers.Count() - 1]; else return 0; } private static Container containerAND(Container ca, Container cb) { BitmapContainer a = null; BitmapContainer b = null; if (ca is BitmapContainer) a = (BitmapContainer)ca; else if (ca is OffsetContainer) a = (BitmapContainer)ca.ToBitmap(); else a = (BitmapContainer)ca.ToBitmap(); if (cb is BitmapContainer) b = (BitmapContainer)cb; else if (cb is OffsetContainer) b = (BitmapContainer)cb.ToBitmap(); else b = (BitmapContainer)cb.ToBitmap(); var av = a.Values(); var bv = b.Values(); var la = av != null ? av.Length : 1024; var lb = bv != null ? bv.Length : 1024; var min = la; if (lb < min) min = lb; List vals = new List(); for (int i = 0; i < min; i++) { ulong ua = ulong.MaxValue; ulong ub = ulong.MaxValue; if (av != null) ua = av[i]; if (bv != null) ub = bv[i]; vals.Add(ua & ub); } return new BitmapContainer(vals.ToArray()); } private static Container containerOR(Container ca, Container cb) { BitmapContainer a = null; BitmapContainer b = null; if (ca is BitmapContainer) a = (BitmapContainer)ca; else if (ca is OffsetContainer) a = (BitmapContainer)ca.ToBitmap(); else a = (BitmapContainer)ca.ToBitmap(); if (cb is BitmapContainer) b = (BitmapContainer)cb; else if (cb is OffsetContainer) b = (BitmapContainer)cb.ToBitmap(); else b = (BitmapContainer)cb.ToBitmap(); var av = a.Values(); var bv = b.Values(); var la = av != null ? av.Length : 1024; var lb = bv != null ? bv.Length : 1024; var max = la; if (lb > max) max = lb; List vals = new List(); for (int i = 0; i < max; i++) { ulong ua = 0; ulong ub = 0; if (av != null && i < la) ua = av[i]; if (bv != null && i < lb) ub = bv[i]; vals.Add(ua | ub); } return new BitmapContainer(vals.ToArray()); } private static unsafe byte[] GetBytes(ushort num, bool reverse) { byte[] buffer = new byte[2]; fixed (byte* numRef = buffer) { *((ushort*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } private static unsafe ulong ToULong(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((ulong*)numRef)); } } private static unsafe ushort ToUShort(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((ushort*)numRef)); } } private static unsafe byte[] ToByteArray(ulong[] data) { int arrayLength = data.Length; byte[] byteArray = new byte[8 * arrayLength]; fixed (ulong* pointer = data) { fixed (byte* bytePointer = byteArray) { ulong* read = pointer; ulong* write = (ulong*)bytePointer; for (int i = 0; i < arrayLength; i++) { *write++ = *read++; } } // below not working //System.Runtime.InteropServices.Marshal.Copy(new IntPtr(pointer), byteArray, 0, arrayLength); } // not working //fixed (ulong* src = data) //{ // System.Runtime.InteropServices.Marshal.Copy(new IntPtr(src), byteArray, 0, arrayLength); //} // not working //Buffer.BlockCopy(data, 0, byteArray, 0, arrayLength); return byteArray; } private static unsafe byte[] ToByteArray(ushort[] data) { int arrayLength = data.Length; byte[] byteArray = new byte[2 * arrayLength]; fixed (ushort* pointer = data) { fixed (byte* bytePointer = byteArray) { ushort* read = pointer; ushort* write = (ushort*)bytePointer; for (int i = 0; i < arrayLength; i++) { *write++ = *read++; } } } // not working //fixed (ushort* src = data) //{ // System.Runtime.InteropServices.Marshal.Copy(new IntPtr(src), byteArray, 0, arrayLength); //} // not working //Buffer.BlockCopy(data, 0, byteArray, 0, arrayLength); return byteArray; } } } ================================================ FILE: RaptorDB/Helper/WAHBitarray2.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; namespace RaptorDB { public class WAHBitArray { public enum TYPE { WAH = 1, Bitarray = 0, Indexes = 2 } public WAHBitArray() { _state = TYPE.Indexes; if (Global.UseLessMemoryStructures) _offsets = new SafeSortedList(); else _offsets = new SafeDictionary(); } public WAHBitArray(TYPE type, uint[] ints) { _state = type; switch (type) { case TYPE.WAH: _compressed = ints; Uncompress(); _state = TYPE.Bitarray; _compressed = null; break; case TYPE.Bitarray: _uncompressed = ints; break; case TYPE.Indexes: if (Global.UseLessMemoryStructures) _offsets = new SafeSortedList(); else _offsets = new SafeDictionary(); //new Dictionary(); foreach (var i in ints) _offsets.Add(i, true); break; } } private uint[] _compressed; private uint[] _uncompressed; //private Dictionary _offsets = new Dictionary(); private IKV _offsets = null;// new SafeSortedList(); private uint _curMax = 0; private TYPE _state; public bool isDirty = false; public WAHBitArray Copy() { lock (_lock) { uint[] i = GetBitArray(); return new WAHBitArray(TYPE.Bitarray, i); } } public bool Get(int index) { lock (_lock) { if (_state == TYPE.Indexes) { bool b = false; var f = _offsets.TryGetValue((uint)index, out b); if (f) return b; else return false; } CheckBitArray(); Resize(index); return internalGet(index); } } private object _lock = new object(); public void Set(int index, bool val) { lock (_lock) { if (_state == TYPE.Indexes) { isDirty = true; if (val == true) { _offsets.Add((uint)index, true); // set max if (index > _curMax) _curMax = (uint)index; } else { _offsets.Remove((uint)index); } ChangeTypeIfNeeded(); return; } CheckBitArray(); Resize(index); internalSet(index, val); } } public int Length { set { lock (_lock) { if (_state == TYPE.Indexes) { // ignore return; } CheckBitArray(); int c = value >> 5; c++; if (c > _uncompressed.Length) { uint[] ar = new uint[c]; _uncompressed.CopyTo(ar, 0); _uncompressed = ar; } } } get { if (_state == TYPE.Indexes) { if (_offsets.Count() == 0) return 0; uint[] k = GetOffsets(); uint l = k[k.Length - 1]; return (int)l; } CheckBitArray(); return _uncompressed.Length << 5; } } #region [ B I T O P E R T A I O N S ] public WAHBitArray And(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) left[i] &= right[i]; return new WAHBitArray(TYPE.Bitarray, left); } } public WAHBitArray AndNot(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) left[i] &= ~right[i]; return new WAHBitArray(TYPE.Bitarray, left); } } public WAHBitArray Or(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) left[i] |= right[i]; return new WAHBitArray(TYPE.Bitarray, left); } } public WAHBitArray Not(int size) { lock (_lock) { this.CheckBitArray(); uint[] left = this.GetBitArray(); int c = left.Length; int ms = size >> 5; if (size - (ms << 5) > 0) ms++; // include remainder if (ms > c) { var a = new uint[ms]; Array.Copy(left, 0, a, 0, c); left = a; c = ms; } for (int i = 0; i < c; i++) left[i] = ~left[i]; return new WAHBitArray(TYPE.Bitarray, left); } } public WAHBitArray Xor(WAHBitArray op) { lock (_lock) { uint[] left; uint[] right; prelogic(op, out left, out right); for (int i = 0; i < left.Length; i++) left[i] ^= right[i]; return new WAHBitArray(TYPE.Bitarray, left); } } #endregion private static int BitCount(uint n) { // 32-bit recursive reduction using SWAR n -= ((n >> 1) & 0x55555555); n = (((n >> 2) & 0x33333333) + (n & 0x33333333)); n = (((n >> 4) + n) & 0x0f0f0f0f); return (int)((n * 0x01010101) >> 24); } public long CountOnes() { if (_state == TYPE.Indexes) { return _offsets.Count(); } long c = 0; CheckBitArray(); foreach (uint i in _uncompressed) c += BitCount(i); return c; } public long CountZeros() { if (_state == TYPE.Indexes) { long ones = _offsets.Count(); uint[] k = GetOffsets(); long l = k[k.Length - 1]; return l - ones; } CheckBitArray(); int count = _uncompressed.Length << 5; long cc = CountOnes(); return count - cc; } public void FreeMemory() { if (_state == TYPE.Bitarray) { if (_uncompressed != null) { lock (_lock) { _compressed = Compress(_uncompressed); _uncompressed = null; _state = TYPE.WAH; } } } } public uint[] GetCompressed(out TYPE type) { type = TYPE.WAH; ChangeTypeIfNeeded(); if (_state == TYPE.Indexes) { type = TYPE.Indexes; return GetOffsets(); } else if (_uncompressed == null) return new uint[] { 0 }; uint[] d = Compress(_uncompressed); return d; } public IEnumerable GetBitIndexes() { if (_state == TYPE.Indexes) { foreach (int i in GetOffsets()) yield return i; } else { CheckBitArray(); int count = _uncompressed.Length; for (int i = 0; i < count; i++) { if (_uncompressed[i] > 0) { for (int j = 0; j < 32; j++) { bool b = internalGet((i << 5) + j); if (b == true)// ones) yield return (i << 5) + j; } } } } } #region [ P R I V A T E ] private uint[] GetOffsets() { uint[] k; lock (_lock) { k = new uint[_offsets.Count()]; _offsets.Keys().CopyTo(k, 0); } Array.Sort(k); return k; } private void prelogic(WAHBitArray op, out uint[] left, out uint[] right) { this.CheckBitArray(); left = this.GetBitArray(); right = op.GetBitArray(); int ic = left.Length; int uc = right.Length; if (ic > uc) { uint[] ar = new uint[ic]; right.CopyTo(ar, 0); right = ar; } else if (ic < uc) { uint[] ar = new uint[uc]; left.CopyTo(ar, 0); left = ar; } } internal uint[] GetBitArray() { lock (_lock) { if (_state == TYPE.Indexes) return UnpackOffsets(); this.CheckBitArray(); uint[] ui = new uint[_uncompressed.Length]; _uncompressed.CopyTo(ui, 0); return ui; } } private uint[] UnpackOffsets() { // return bitmap uints uint max = 0; if (_offsets.Count() == 0) return new uint[0]; uint[] k = GetOffsets(); max = k[k.Length - 1]; uint[] ints = new uint[(max >> 5) + 1]; foreach (int index in k) { int pointer = ((int)index) >> 5; uint mask = (uint)1 << (31 - // high order bit set ((int)index % 32)); ints[pointer] |= mask; } return ints; } private void ChangeTypeIfNeeded() { if (_state != TYPE.Indexes) return; uint T = (_curMax >> 5) + 1; int c = _offsets.Count(); if (c > T && c > Global.BitmapOffsetSwitchOverCount) { // change type to WAH _state = TYPE.Bitarray; _uncompressed = new uint[0]; // create bitmap foreach (var i in _offsets.Keys()) Set((int)i, true); // clear list if (Global.UseLessMemoryStructures) _offsets = new SafeSortedList(); else _offsets = new SafeDictionary(); //new Dictionary(); } } private void Resize(int index) { if (_state == TYPE.Indexes) return; int c = index >> 5; c++; if (_uncompressed == null) { _uncompressed = new uint[c]; return; } if (c > _uncompressed.Length) { uint[] ar = new uint[c]; _uncompressed.CopyTo(ar, 0); _uncompressed = ar; } } private static void ResizeAsNeeded(List list, int index) { int count = index >> 5; while (list.Count < count) list.Add(0); } private void internalSet(int index, bool val) { isDirty = true; int pointer = index >> 5; uint mask = (uint)1 << (31 - // high order bit set (index % 32)); if (val) _uncompressed[pointer] |= mask; else _uncompressed[pointer] &= ~mask; } private bool internalGet(int index) { int pointer = index >> 5; uint mask = (uint)1 << (31 - // high order bit get (index % 32)); if (pointer < _uncompressed.Length) return (_uncompressed[pointer] & mask) != 0; else return false; } private void CheckBitArray() { if (_state == TYPE.Bitarray) return; if (_state == TYPE.WAH) { _uncompressed = new uint[0]; Uncompress(); _state = TYPE.Bitarray; _compressed = null; return; } } #region compress / uncompress private static uint Take31Bits(uint[] data, int index) { ulong l1 = 0; ulong l2 = 0; ulong l = 0; ulong ret = 0; int off = (index % 32); int pointer = index >> 5; l1 = data[pointer]; pointer++; if (pointer < data.Length) l2 = data[pointer]; l = (l1 << 32) + l2; ret = (l >> (33 - off)) & 0x7fffffff; return (uint)ret; } private static uint[] Compress(uint[] data) { List compressed = new List(); uint zeros = 0; uint ones = 0; int count = data.Length << 5; int i = 0; while (i < count)//for (int i = 0; i < count;) { uint num = Take31Bits(data, i); i += 31; if (num == 0) // all zero { zeros += 31; FlushOnes(compressed, ref ones); } else if (num == 0x7fffffff) // all ones { ones += 31; FlushZeros(compressed, ref zeros); } else // literal { FlushOnes(compressed, ref ones); FlushZeros(compressed, ref zeros); compressed.Add(num); } } FlushOnes(compressed, ref ones); FlushZeros(compressed, ref zeros); return compressed.ToArray(); } private static void FlushOnes(List compressed, ref uint ones) { if (ones > 0) { uint n = 0xc0000000 + ones; ones = 0; compressed.Add(n); } } private static void FlushZeros(List compressed, ref uint zeros) { if (zeros > 0) { uint n = 0x80000000 + zeros; zeros = 0; compressed.Add(n); } } private static void Write31Bits(List list, int index, uint val) { ResizeAsNeeded(list, index + 32); int off = (index % 32); int pointer = index >> 5; if (pointer >= list.Count - 1) list.Add(0); ulong l = ((ulong)list[pointer] << 32) + list[pointer + 1]; l |= (ulong)val << (33 - off); list[pointer] = (uint)(l >> 32); list[pointer + 1] = (uint)(l & 0xffffffff); } private void WriteOnes(List list, int index, uint count) { ResizeAsNeeded(list, index); int off = index % 32; int pointer = index >> 5; int ccount = (int)count; int indx = index; int x = 32 - off; if (pointer >= list.Count) list.Add(0); if (ccount > x)//|| x == 32) //current pointer { list[pointer] |= (uint)((0xffffffff >> off)); ccount -= x; indx += x; } else { list[pointer] |= (uint)((0xffffffff << (32 - ccount)) >> off); ccount = 0; } bool checklast = true; while (ccount >= 32)//full ints { if (checklast && list[list.Count - 1] == 0) { list.RemoveAt(list.Count - 1); checklast = false; } list.Add(0xffffffff); ccount -= 32; indx += 32; } int p = indx >> 5; off = indx % 32; if (ccount > 0) { uint i = 0xffffffff << (32 - ccount); if (p > (list.Count - 1)) //remaining list.Add(i); else list[p] |= (uint)(i >> off); } } private void Uncompress() { int index = 0; List list = new List(); if (_compressed == null) return; foreach (uint ci in _compressed) { if ((ci & 0x80000000) == 0) // literal { Write31Bits(list, index, ci); index += 31; } else { uint count = ci & 0x3fffffff; if ((ci & 0x40000000) != 0) // ones count WriteOnes(list, index, count); index += (int)count; } } ResizeAsNeeded(list, index); _uncompressed = list.ToArray(); } #endregion #endregion internal static WAHBitArray Fill(int count) { if (count > 0) { int c = count >> 5; int r = count % 32; if (r > 0) c++; uint[] ints = new uint[c]; for (int i = 0; i < c; i++) ints[i] = 0xffffffff; if (r > 0) ints[c - 1] = 0xffffffff << (31 - r); return new WAHBitArray(TYPE.Bitarray, ints); } return new WAHBitArray(); } internal int GetFirst() { foreach (var i in GetBitIndexes()) return i; return 0; } } } ================================================ FILE: RaptorDB/Indexes/BitmapIndex.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.IO; using System.Threading; namespace RaptorDB { internal class BitmapIndex { public BitmapIndex(string path, string filename) { if (Global.UseLessMemoryStructures) _cache = new SafeSortedList(); else _cache = new SafeDictionary(); _FileName = Path.GetFileNameWithoutExtension(filename); _Path = path; if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) _Path += Path.DirectorySeparatorChar.ToString(); Initialize(); } class L : IDisposable { BitmapIndex _sc; public L(BitmapIndex sc) { _sc = sc; _sc.CheckInternalOP(); } void IDisposable.Dispose() { _sc.Done(); } } private string _recExt = ".mgbmr"; private string _bmpExt = ".mgbmp"; private string _FileName = ""; private string _Path = ""; private FileStream _bitmapFileWriteOrg; private BufferedStream _bitmapFileWrite; private FileStream _bitmapFileRead; private FileStream _recordFileRead; private FileStream _recordFileWriteOrg; private BufferedStream _recordFileWrite; private long _lastBitmapOffset = 0; private int _lastRecordNumber = 0; //private SafeDictionary _cache = new SafeDictionary(); private IKV _cache = null;// new SafeSortedList(); private ILog log = LogManager.GetLogger(typeof(BitmapIndex)); private bool _stopOperations = false; private bool _shutdownDone = false; private int _workingCount = 0; private bool _isDirty = false; #region public void Shutdown() { using (new L(this)) { log.Debug("Shutdown BitmapIndex"); InternalShutdown(); } } public int GetFreeRecordNumber() { using (new L(this)) { int i = _lastRecordNumber++; _cache.Add(i, new MGRB()); return i; } } public void Commit(bool freeMemory) { if (_isDirty == false) return; using (new L(this)) { log.Debug("writing " + _FileName); int[] keys = _cache.Keys(); Array.Sort(keys); foreach (int k in keys) { MGRB bmp = null; if (_cache.TryGetValue(k, out bmp) && bmp.isDirty) { bmp.Optimize(); SaveBitmap(k, bmp); bmp.isDirty = false; } } Flush(); if (freeMemory) { if (Global.UseLessMemoryStructures) _cache = new SafeSortedList(); else _cache = new SafeDictionary(); log.Debug(" freeing cache"); } _isDirty = false; } } public void SetDuplicate(int bitmaprecno, int record) { using (new L(this)) { MGRB ba = null; ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno); ba.Set(record, true); _isDirty = true; } } public MGRB GetBitmap(int recno) { using (new L(this)) { return internalGetBitmap(recno); } } private object _oplock = new object(); public void Optimize() { lock (_oplock) lock (_readlock) lock (_writelock) { _stopOperations = true; while (_workingCount > 0) Thread.SpinWait(1); Flush(); if (File.Exists(_Path + _FileName + "$" + _bmpExt)) File.Delete(_Path + _FileName + "$" + _bmpExt); if (File.Exists(_Path + _FileName + "$" + _recExt)) File.Delete(_Path + _FileName + "$" + _recExt); Stream _newrec = new FileStream(_Path + _FileName + "$" + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); Stream _newbmp = new FileStream(_Path + _FileName + "$" + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); long newoffset = 0; int c = (int)(_recordFileRead.Length / 8); for (int i = 0; i < c; i++) { long offset = ReadRecordOffset(i); byte[] b = ReadBMPDataForOptimize(offset); if (b == null) { _stopOperations = false; throw new Exception("bitmap index file is corrupted"); } _newrec.Write(Helper.GetBytes(newoffset, false), 0, 8); newoffset += b.Length; _newbmp.Write(b, 0, b.Length); } _newbmp.Flush(); _newbmp.Close(); _newrec.Flush(); _newrec.Close(); InternalShutdown(); File.Delete(_Path + _FileName + _bmpExt); File.Delete(_Path + _FileName + _recExt); File.Move(_Path + _FileName + "$" + _bmpExt, _Path + _FileName + _bmpExt); File.Move(_Path + _FileName + "$" + _recExt, _Path + _FileName + _recExt); Initialize(); _stopOperations = false; } } internal void FreeMemory() { try { List free = new List(); foreach (var k in _cache.Keys()) { var val = _cache.GetValue(k); if (val.isDirty == false) free.Add(k); } log.Info("releasing bmp count = " + free.Count + " out of " + _cache.Count()); foreach (int i in free) _cache.Remove(i); } catch (Exception ex) { log.Error(ex); } } #endregion #region [ P R I V A T E ] private long ReadRecordOffset(int recnum) { byte[] b = new byte[8]; long off = ((long)recnum) * 8; _recordFileRead.Seek(off, SeekOrigin.Begin); _recordFileRead.Read(b, 0, 8); return Helper.ToInt64(b, 0); } private void Initialize() { _recordFileRead = new FileStream(_Path + _FileName + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); _recordFileWriteOrg = new FileStream(_Path + _FileName + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); _recordFileWrite = new BufferedStream(_recordFileWriteOrg); _bitmapFileRead = new FileStream(_Path + _FileName + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); _bitmapFileWriteOrg = new FileStream(_Path + _FileName + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite); _bitmapFileWrite = new BufferedStream(_bitmapFileWriteOrg); _bitmapFileWrite.Seek(0L, SeekOrigin.End); _lastBitmapOffset = _bitmapFileWrite.Length; _lastRecordNumber = (int)(_recordFileRead.Length / 8); _shutdownDone = false; } private void InternalShutdown() { bool d1 = false; bool d2 = false; if (_shutdownDone == false) { Flush(); if (_recordFileWrite.Length == 0) d1 = true; if (_bitmapFileWrite.Length == 0) d2 = true; _recordFileRead.Close(); _bitmapFileRead.Close(); _bitmapFileWriteOrg.Close(); _recordFileWriteOrg.Close(); _recordFileWrite.Close(); _bitmapFileWrite.Close(); if (d1) File.Delete(_Path + _FileName + _recExt); if (d2) File.Delete(_Path + _FileName + _bmpExt); _recordFileWrite = null; _recordFileRead = null; _bitmapFileRead = null; _bitmapFileWrite = null; _recordFileRead = null; _recordFileWrite = null; _shutdownDone = true; } } private void Flush() { if (_shutdownDone) return; if (_recordFileWrite != null) _recordFileWrite.Flush(); if (_bitmapFileWrite != null) _bitmapFileWrite.Flush(); if (_recordFileRead != null) _recordFileRead.Flush(); if (_bitmapFileRead != null) _bitmapFileRead.Flush(); if (_bitmapFileWriteOrg != null) _bitmapFileWriteOrg.Flush(); if (_recordFileWriteOrg != null) _recordFileWriteOrg.Flush(); } private object _readlock = new object(); private MGRB internalGetBitmap(int recno) { lock (_readlock) { MGRB ba = new MGRB(); if (recno == -1) return ba; if (_cache.TryGetValue(recno, out ba)) { return ba; } else { long offset = 0; //if (_offsetCache.TryGetValue(recno, out offset) == false) { offset = ReadRecordOffset(recno); // _offsetCache.Add(recno, offset); } ba = LoadBitmap(offset); _cache.Add(recno, ba); return ba; } } } private object _writelock = new object(); private void SaveBitmap(int recno, MGRB bmp) { lock (_writelock) { long offset = SaveBitmapToFile(bmp); //long v; //if (_offsetCache.TryGetValue(recno, out v)) // _offsetCache[recno] = offset; //else // _offsetCache.Add(recno, offset); long pointer = ((long)recno) * 8; _recordFileWrite.Seek(pointer, SeekOrigin.Begin); byte[] b = new byte[8]; b = Helper.GetBytes(offset, false); _recordFileWrite.Write(b, 0, 8); } } //----------------------------------------------------------------- // new format // 0 : b // 1 : m // 2 : type 0 = uncompressed, 1 = compressed // 3 : data size (int) // 8 : data bytes private byte _hdrlen = 2 + 4 + 1; private long SaveBitmapToFile(MGRB bmp) { long off = _lastBitmapOffset; var dat = bmp.Serialize(); var hdr = new byte[_hdrlen]; var b = fastBinaryJSON.BJSON.ToBJSON(dat, new fastBinaryJSON.BJSONParameters { UseExtensions = false }); hdr[0] = (byte)'b'; hdr[1] = (byte)'m'; hdr[2] = 0; // uncompressed if (Global.CompressBitmapBytes) { hdr[2] = 1; b = MiniLZO.Compress(b); } var s = Helper.GetBytes(b.Length, false); Buffer.BlockCopy(s, 0, hdr, 3, 4); _bitmapFileWrite.Write(hdr, 0, hdr.Length); _lastBitmapOffset += hdr.Length; _bitmapFileWrite.Write(b, 0, b.Length); _lastBitmapOffset += b.Length; return off; } private byte[] ReadBMPDataForOptimize(long offset) { // return data + header _bitmapFileRead.Seek(offset, SeekOrigin.Begin); byte[] hdr = new byte[_hdrlen]; _bitmapFileRead.Read(hdr, 0, _hdrlen); if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m') { int c = Helper.ToInt32(hdr, 3); var data = new byte[c + _hdrlen]; Buffer.BlockCopy(hdr, 0, data, 0, _hdrlen); _bitmapFileRead.Read(data, _hdrlen, c); return data; } return null; } private MGRB LoadBitmap(long offset) { MGRB bc = new MGRB(); if (offset == -1) return bc; FileStream bmp = _bitmapFileRead; bmp.Seek(offset, SeekOrigin.Begin); var hdr = new byte[_hdrlen]; bmp.Read(hdr, 0, hdr.Length); if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m') { int c = Helper.ToInt32(hdr, 3); var b = new byte[c]; bmp.Read(b, 0, c); if (hdr[2] == 1) b = MiniLZO.Decompress(b); bc.Deserialize(fastBinaryJSON.BJSON.ToObject(b)); } else log.Error("bitmap not recognized"); return bc; } #pragma warning disable 642 private void CheckInternalOP() { if (_stopOperations) lock (_oplock) { } // yes! this is good Interlocked.Increment(ref _workingCount); } #pragma warning restore 642 private void Done() { Interlocked.Decrement(ref _workingCount); } #endregion } } ================================================ FILE: RaptorDB/Indexes/Cache.cs ================================================ using System; using System.Xml.Serialization; namespace RaptorDB { public enum OPERATION { AND, OR, ANDNOT } public class Document { public string FullName; public DateTime Created; public DateTime Modified; public long Length; public string Extension; public Document() { DocNumber = -1; } public Document(string filename, string text) { FileName = filename; Text = text; DocNumber = -1; } public int DocNumber { get; set; } [XmlIgnore] public string Text { get; set; } public string FileName { get; set; } public string Abstract { get; set; } public override string ToString() { return FileName; } } } ================================================ FILE: RaptorDB/Indexes/Hoot.cs ================================================ using System; using System.Collections.Generic; using System.Text; using System.IO; using System.Text.RegularExpressions; using RaptorDB.Common; namespace RaptorDB { public class Hoot { public Hoot(string IndexPath, string FileName, bool DocMode) : this(IndexPath, FileName, DocMode, new tokenizer()) { } public Hoot(string IndexPath, string FileName, bool DocMode, ITokenizer tokenizer) { if (tokenizer != null) _tokenizer = tokenizer; else _tokenizer = new tokenizer(); _Path = IndexPath; _FileName = FileName; _docMode = DocMode; if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) _Path += Path.DirectorySeparatorChar; Directory.CreateDirectory(IndexPath); _log.Debug("Starting hOOt...."); _log.Debug("Storage Folder = " + _Path); if (DocMode) { _docs = new KeyStoreString(_Path + "files.docs", false); // read deleted _deleted = new BoolIndex(_Path, "_deleted", ".hoot"); _lastDocNum = (int)_docs.Count(); } _bitmaps = new BitmapIndex(_Path, _FileName + "_hoot.bmp"); // read words LoadWords(); } private ITokenizer _tokenizer; private SafeDictionary _words = new SafeDictionary(); //private SafeSortedList _words = new SafeSortedList(); private BitmapIndex _bitmaps; private BoolIndex _deleted; private ILog _log = LogManager.GetLogger(typeof(Hoot)); private int _lastDocNum = 0; private string _FileName = "words"; private string _Path = ""; private KeyStoreString _docs; private bool _docMode = false; private bool _wordschanged = true; private bool _shutdowndone = false; private object _lock = new object(); public string[] Words { get { checkloaded(); return _words.Keys(); } } public int WordCount { get { checkloaded(); return _words.Count(); } } public int DocumentCount { get { checkloaded(); return _lastDocNum - (int)_deleted.GetBits().CountOnes(); } } public string IndexPath { get { return _Path; } } public void Save() { lock (_lock) InternalSave(); } public void Index(int recordnumber, string text) { checkloaded(); AddtoIndex(recordnumber, text); } public MGRB Query(string filter, int maxsize) { checkloaded(); return ExecutionPlan(filter, maxsize); } public int Index(Document doc, bool deleteold) { checkloaded(); _log.Info("indexing doc : " + doc.FileName); DateTime dt = FastDateTime.Now; if (deleteold && doc.DocNumber > -1) _deleted.Set(true, doc.DocNumber); if (deleteold == true || doc.DocNumber == -1) doc.DocNumber = _lastDocNum++; // save doc to disk string dstr = fastJSON.JSON.ToJSON(doc, new fastJSON.JSONParameters { UseExtensions = false }); _docs.Set(doc.FileName.ToLower(), fastJSON.Reflection.UnicodeGetBytes(dstr)); _log.Info("writing doc to disk (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); dt = FastDateTime.Now; // index doc AddtoIndex(doc.DocNumber, doc.Text); _log.Info("indexing time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return _lastDocNum; } public IEnumerable FindRows(string filter) { checkloaded(); MGRB bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate records return bits.GetBitIndexes(); } public IEnumerable FindDocuments(string filter) { checkloaded(); MGRB bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) break; string b = _docs.ReadData(i); T d = fastJSON.JSON.ToObject(b, new fastJSON.JSONParameters { ParametricConstructorOverride = true }); yield return d; } } public IEnumerable FindDocumentFileNames(string filter) { checkloaded(); MGRB bits = ExecutionPlan(filter, _docs.RecordCount()); // enumerate documents foreach (int i in bits.GetBitIndexes()) { if (i > _lastDocNum - 1) break; string b = _docs.ReadData(i); var d = (Dictionary)fastJSON.JSON.Parse(b); yield return d["FileName"].ToString(); } } public void RemoveDocument(int number) { // add number to deleted bitmap _deleted.Set(true, number); } public bool RemoveDocument(string filename) { // remove doc based on filename byte[] b; if (_docs.Get(filename.ToLower(), out b)) { Document d = fastJSON.JSON.ToObject(fastJSON.Reflection.UnicodeGetString(b)); RemoveDocument(d.DocNumber); return true; } return false; } public bool IsIndexed(string filename) { byte[] b; return _docs.Get(filename.ToLower(), out b); } public void OptimizeIndex() { lock (_lock) { InternalSave(); //_bitmaps.Commit(false); _bitmaps.Optimize(); } } #region [ P R I V A T E M E T H O D S ] private void checkloaded() { if (_wordschanged == false) { LoadWords(); } } private MGRB ExecutionPlan(string filter, int maxsize) { //_log.Debug("query : " + filter); DateTime dt = FastDateTime.Now; // query indexes string[] words = filter.Split(' '); //bool defaulttoand = true; //if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0) // defaulttoand = false; MGRB found = null;// MGRB.Fill(maxsize); foreach (string s in words) { int c; bool not = false; string word = s; if (s == "") continue; OPERATION op = OPERATION.AND; //if (defaulttoand) // op = OPERATION.AND; if (word.StartsWith("+")) { op = OPERATION.OR; word = s.Replace("+", ""); } if (word.StartsWith("-")) { op = OPERATION.ANDNOT; word = s.Replace("-", ""); not = true; if (found == null) // leading with - -> "-oak hill" { found = MGRB.Fill(maxsize); } } if (word.Contains("*") || word.Contains("?")) { MGRB wildbits = new MGRB(); // do wildcard search Regex reg = new Regex("^" + word.Replace("*", ".*").Replace("?", ".") + "$", RegexOptions.IgnoreCase); foreach (string key in _words.Keys()) { if (reg.IsMatch(key)) { _words.TryGetValue(key, out c); MGRB ba = _bitmaps.GetBitmap(c); wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize); } } if (found == null) found = wildbits; else { if (not) // "-oak -*l" found = found.AndNot(wildbits); else if (op == OPERATION.AND) found = found.And(wildbits); else found = found.Or(wildbits); } } else if (_words.TryGetValue(word.ToLowerInvariant(), out c)) { // bits logic MGRB ba = _bitmaps.GetBitmap(c); found = DoBitOperation(found, ba, op, maxsize); } else if (op == OPERATION.AND) found = new MGRB(); } if (found == null) return new MGRB(); // remove deleted docs MGRB ret; if (_docMode) ret = found.AndNot(_deleted.GetBits()); else ret = found; //_log.Debug("query time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return ret; } private static MGRB DoBitOperation(MGRB bits, MGRB c, OPERATION op, int maxsize) { if (bits != null) { switch (op) { case OPERATION.AND: bits = bits.And(c); break; case OPERATION.OR: bits = bits.Or(c); break; case OPERATION.ANDNOT: bits = bits.And(c.Not(maxsize)); break; } } else bits = c; return bits; } private void InternalSave() { _log.Info("saving index..."); DateTime dt = FastDateTime.Now; // save deleted if (_deleted != null) _deleted.SaveIndex(); // save docs if (_docMode) _docs.SaveIndex(); if (_bitmaps != null) _bitmaps.Commit(true); if (_words != null && _wordschanged == true) { // save words and bitmaps using (FileStream words = new FileStream(_Path + _FileName + ".words", FileMode.Create)) { using (BinaryWriter bw = new BinaryWriter(words, Encoding.UTF8)) { foreach (string key in _words.Keys()) { bw.Write(key); bw.Write(_words[key]); } } } _wordschanged = false; } _log.Info("save time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); } private void LoadWords() { lock (_lock) { if (_words == null) _words = new SafeDictionary(); // new SafeSortedList(); if (File.Exists(_Path + _FileName + ".words") == false) return; // load words using (FileStream words = new FileStream(_Path + _FileName + ".words", FileMode.Open)) { if (words.Length == 0) return; using (BinaryReader br = new BinaryReader(words, Encoding.UTF8)) { string s = br.ReadString(); while (s != "") { int off = br.ReadInt32(); _words.Add(s, off); try { s = br.ReadString(); } catch { s = ""; } } } } //byte[] b = File.ReadAllBytes(_Path + _FileName + ".words"); //if (b.Length == 0) // return; //MemoryStream ms = new MemoryStream(b); //BinaryReader br = new BinaryReader(ms, Encoding.UTF8); //string s = br.ReadString(); //while (s != "") //{ // int off = br.ReadInt32(); // _words.Add(s, off); // try // { // s = br.ReadString(); // } // catch { s = ""; } //} _log.Debug("Word Count = " + _words.Count()); _wordschanged = true; } } private void AddtoIndex(int recnum, string text) { if (text == "" || text == null) return; text = text.ToLowerInvariant(); // lowercase index string[] keys; if (_docMode) { //_log.Debug("text size = " + text.Length); Dictionary wordfreq = _tokenizer.GenerateWordFreq(text); //_log.Debug("word count = " + wordfreq.Count); var kk = wordfreq.Keys; keys = new string[kk.Count]; kk.CopyTo(keys, 0); } else { keys = text.Split(' '); } foreach (string key in keys) { if (key == "") continue; int bmp; if (_words.TryGetValue(key, out bmp)) { _bitmaps.GetBitmap(bmp).Set(recnum, true); } else { bmp = _bitmaps.GetFreeRecordNumber(); _bitmaps.SetDuplicate(bmp, recnum); _words.Add(key, bmp); } } _wordschanged = true; } #endregion public void Shutdown() { lock (_lock) { if (_shutdowndone == true) return; InternalSave(); if (_deleted != null) { _deleted.SaveIndex(); _deleted.Shutdown(); _deleted = null; } if (_bitmaps != null) { _bitmaps.Commit(Global.FreeBitmapMemoryOnSave); _bitmaps.Shutdown(); _bitmaps = null; } if (_docMode) _docs.Shutdown(); _shutdowndone = true; } } public void FreeMemory() { lock (_lock) { InternalSave(); if (_deleted != null) _deleted.FreeMemory(); if (_bitmaps != null) _bitmaps.FreeMemory(); if (_docs != null) _docs.FreeMemory(); //_words = null;// new SafeSortedList(); //_loaded = false; } } public T Fetch(int docnum) { string b = _docs.ReadData(docnum); return fastJSON.JSON.ToObject(b); } } } ================================================ FILE: RaptorDB/Indexes/IIndex.cs ================================================ namespace RaptorDB { internal enum RDBExpression { Equal, Greater, GreaterEqual, Less, LessEqual, NotEqual, Between, Contains } internal interface IIndex { void Set(object key, int recnum); MGRB Query(object fromkey, object tokey, int maxsize); MGRB Query(RDBExpression ex, object from , int maxsize); void FreeMemory(); void Shutdown(); void SaveIndex(); object[] GetKeys(); } } ================================================ FILE: RaptorDB/Indexes/ITokenizer.cs ================================================ using System.Collections.Generic; namespace RaptorDB { public interface ITokenizer { Dictionary GenerateWordFreq(string text); } } ================================================ FILE: RaptorDB/Indexes/IndexFile.cs ================================================ using System; using System.Collections.Generic; using System.IO; using RaptorDB.Common; using System.Threading; using fastBinaryJSON; namespace RaptorDB { internal class IndexFile { FileStream _file = null; private byte[] _FileHeader = new byte[] { (byte)'M', (byte)'G', (byte)'I', 0, // 3 = [keysize] max 255 0,0, // 4 = [node size] max 65536 0,0,0,0, // 6 = [root page num] 0, // 10 = Index file type : 0=mgindex 1=mgindex+strings (key = firstallocblock) 0,0,0,0 // 11 = last record number indexed }; private byte[] _BlockHeader = new byte[] { (byte)'P',(byte)'A',(byte)'G',(byte)'E', 0, // 4 = [Flag] = 0=page 1=page list 0,0, // 5 = [item count] 0,0,0,0, // 7 = reserved 0,0,0,0 // 11 = [right page number] / [next page number] }; internal byte _maxKeySize; internal ushort _PageNodeCount = 5000; private int _LastPageNumber = 1; // 0 = page list private int _PageLength; private int _rowSize; private bool _allowDups = true; ILog log = LogManager.GetLogger(typeof(IndexFile)); private BitmapIndex _bitmap; IGetBytes _T = null; private object _fileLock = new object(); private StringHF _strings; private bool _externalStrings = false; //private List _pagelistalllocblock = null; private string _FileName = ""; public IndexFile(string filename, byte maxKeySize)//, ushort pageNodeCount) { _T = RDBDataType.ByteHandler(); if (typeof(T) == typeof(string) && Global.EnableOptimizedStringIndex) { _externalStrings = true; _maxKeySize = 4;// blocknum:int } else _maxKeySize = maxKeySize; _PageNodeCount = Global.PageItemCount;// pageNodeCount; _rowSize = (_maxKeySize + 1 + 4 + 4); _FileName = filename.Substring(0, filename.LastIndexOf('.')); string path = Path.GetDirectoryName(filename); Directory.CreateDirectory(path); if (File.Exists(filename)) { // if file exists open and read header _file = File.Open(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); ReadFileHeader(); if (_externalStrings == false)// if the file says different { _rowSize = (_maxKeySize + 1 + 4 + 4); } // compute last page number from file length _PageLength = (_BlockHeader.Length + _rowSize * (_PageNodeCount)); _LastPageNumber = (int)((_file.Length - _FileHeader.Length) / _PageLength); } else { // else create new file _file = File.Open(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); _PageLength = (_BlockHeader.Length + _rowSize * (_PageNodeCount)); CreateFileHeader(0); _LastPageNumber = (int)((_file.Length - _FileHeader.Length) / _PageLength); } if (_externalStrings) { _strings = new StringHF(path, Path.GetFileNameWithoutExtension(filename) + ".strings"); } if (_LastPageNumber == 0) _LastPageNumber = 1; // bitmap duplicates if (_allowDups) _bitmap = new BitmapIndex(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename)); } #region [ C o m m o n ] public void SetBitmapDuplicate(int bitmaprec, int rec) { _bitmap.SetDuplicate(bitmaprec, rec); } public int GetBitmapDuplaicateFreeRecordNumber() { return _bitmap.GetFreeRecordNumber(); } public IEnumerable GetDuplicatesRecordNumbers(int recno) { return GetDuplicateBitmap(recno).GetBitIndexes(); } public MGRB GetDuplicateBitmap(int recno) { return _bitmap.GetBitmap(recno); } private byte[] CreateBlockHeader(byte type, ushort itemcount, int rightpagenumber) { byte[] block = new byte[_BlockHeader.Length]; Array.Copy(_BlockHeader, block, block.Length); block[4] = type; byte[] b = Helper.GetBytes(itemcount, false); Buffer.BlockCopy(b, 0, block, 5, 2); b = Helper.GetBytes(rightpagenumber, false); Buffer.BlockCopy(b, 0, block, 11, 4); return block; } private void CreateFileHeader(int rowsindexed) { lock (_fileLock) { // max key size byte[] b = Helper.GetBytes(_maxKeySize, false); Buffer.BlockCopy(b, 0, _FileHeader, 3, 1); // page node count b = Helper.GetBytes(_PageNodeCount, false); Buffer.BlockCopy(b, 0, _FileHeader, 4, 2); b = Helper.GetBytes(rowsindexed, false); Buffer.BlockCopy(b, 0, _FileHeader, 11, 4); if (_externalStrings) _FileHeader[10] = 1; _file.Seek(0L, SeekOrigin.Begin); _file.Write(_FileHeader, 0, _FileHeader.Length); if (rowsindexed == 0) { byte[] pagezero = new byte[_PageLength]; byte[] block = CreateBlockHeader(1, 0, -1); Buffer.BlockCopy(block, 0, pagezero, 0, block.Length); _file.Write(pagezero, 0, _PageLength); } _file.Flush(); } } private bool ReadFileHeader() { _file.Seek(0L, SeekOrigin.Begin); byte[] b = new byte[_FileHeader.Length]; _file.Read(b, 0, _FileHeader.Length); if (b[0] == _FileHeader[0] && b[1] == _FileHeader[1] && b[2] == _FileHeader[2]) // header { byte maxks = b[3]; ushort nodes = (ushort)Helper.ToInt16(b, 4); int root = Helper.ToInt32(b, 6); _maxKeySize = maxks; _PageNodeCount = nodes; _FileHeader = b; if (b[10] == 0) _externalStrings = false; } return false; } public int GetNewPageNumber() { return Interlocked.Increment(ref _LastPageNumber); //_LastPageNumber++; } private void SeekPage(int pnum) { long offset = _FileHeader.Length; offset += (long)pnum * _PageLength; if (offset > _file.Length) CreateBlankPages(pnum); _file.Seek(offset, SeekOrigin.Begin); } private void CreateBlankPages(int pnum) { // create space byte[] b = new byte[_PageLength]; _file.Seek(0L, SeekOrigin.Current); for (int i = pnum; i < _LastPageNumber; i++) _file.Write(b, 0, b.Length); _file.Flush(); } public void FreeMemory() { if (_allowDups) _bitmap.FreeMemory(); } public void Shutdown() { log.Debug("Shutdown IndexFile"); if (_externalStrings) _strings.Shutdown(); if (_file != null) { _file.Flush(); _file.Close(); } _file = null; if (_allowDups) { _bitmap.Commit(Global.FreeBitmapMemoryOnSave); _bitmap.Shutdown(); } } #endregion #region [ P a g e s ] public void GetPageList(List PageListDiskPages, SafeSortedList PageList, out int lastIndexedRow) { lastIndexedRow = Helper.ToInt32(_FileHeader, 11); // load page list PageListDiskPages.Add(0); // first page list int nextpage = LoadPageListData(0, PageList); while (nextpage != -1) { nextpage = LoadPageListData(nextpage, PageList); if (nextpage != -1) PageListDiskPages.Add(nextpage); } } private int LoadPageListData(int page, SafeSortedList PageList) { lock (_fileLock) { // load page list data int nextpage = -1; SeekPage(page); byte[] b = new byte[_PageLength]; _file.Read(b, 0, _PageLength); if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3]) { short count = Helper.ToInt16(b, 5); if (count > _PageNodeCount) throw new Exception("Count > node size"); nextpage = Helper.ToInt32(b, 11); int index = _BlockHeader.Length; object[] keys = null; // TODO : needed?? //if (File.Exists(_FileName + ".pagelist")) //{ // var bn = File.ReadAllBytes(_FileName + ".pagelist"); // int blknum = Helper.ToInt32(bn, 0); // byte[] bb = _strings.GetData(blknum, out _pagelistalllocblock); // keys = (object[])BJSON.ToObject(bb); //} for (int i = 0; i < count; i++) { int idx = index + _rowSize * i; byte ks = b[idx]; T key; if (_externalStrings == false) key = _T.GetObject(b, idx + 1, ks); else { if (keys == null) key = _T.GetObject(b, idx + 1, ks); // do old way until better way else key = (T)keys[i]; } int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize); // add counts int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4); // FEATURE : add dup count PageList.Add(key, new PageInfo(pagenum, unique, 0)); } } else throw new Exception("Page List header is invalid"); return nextpage; } } internal void SavePage(Page page) { lock (_fileLock) { int pnum = page.DiskPageNumber; if (pnum > _LastPageNumber) throw new Exception("should not be here: page out of bounds"); SeekPage(pnum); byte[] pagebytes = new byte[_PageLength]; byte[] blockheader = CreateBlockHeader(0, (ushort)page.tree.Count(), page.RightPageNumber); Buffer.BlockCopy(blockheader, 0, pagebytes, 0, blockheader.Length); int index = blockheader.Length; int i = 0; byte[] b = null; T[] keys = page.tree.Keys(); Array.Sort(keys); // sort keys on save for read performance int blocknum = 0; if (_externalStrings) { // free old blocks if (page.allocblocks != null) _strings.FreeBlocks(page.allocblocks); List blocks = new List(); blocknum = _strings.SaveData(page.DiskPageNumber.ToString(), BJSON.ToBJSON(keys, new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }), out blocks); page.allocblocks = blocks; } // node children foreach (var kp in keys) { var val = page.tree[kp]; int idx = index + _rowSize * i; // key bytes byte[] kk; byte size; if (_externalStrings == false) { kk = _T.GetBytes(kp); size = (byte)kk.Length; if (size > _maxKeySize) size = _maxKeySize; } else { kk = new byte[4]; Buffer.BlockCopy(Helper.GetBytes(blocknum, false), 0, kk, 0, 4); size = 4; } // key size = 1 byte pagebytes[idx] = size; Buffer.BlockCopy(kk, 0, pagebytes, idx + 1, pagebytes[idx]); // offset = 4 bytes b = Helper.GetBytes(val.RecordNumber, false); Buffer.BlockCopy(b, 0, pagebytes, idx + 1 + _maxKeySize, b.Length); // duplicatepage = 4 bytes b = Helper.GetBytes(val.DuplicateBitmapNumber, false); Buffer.BlockCopy(b, 0, pagebytes, idx + 1 + _maxKeySize + 4, b.Length); i++; } _file.Write(pagebytes, 0, pagebytes.Length); } } public Page LoadPageFromPageNumber(int number) { lock (_fileLock) { SeekPage(number); byte[] b = new byte[_PageLength]; _file.Read(b, 0, _PageLength); if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3]) { // create node here Page page = new Page(); short count = Helper.ToInt16(b, 5); if (count > _PageNodeCount) throw new Exception("Count > node size"); page.DiskPageNumber = number; page.RightPageNumber = Helper.ToInt32(b, 11); int index = _BlockHeader.Length; object[] keys = null; for (int i = 0; i < count; i++) { int idx = index + _rowSize * i; byte ks = b[idx]; T key = default(T); if (_externalStrings == false) key = _T.GetObject(b, idx + 1, ks); else { if (keys == null) { int blknum = Helper.ToInt32(b, idx + 1, false); List ablocks = new List(); byte[] bb = _strings.GetData(blknum, out ablocks); page.allocblocks = ablocks; keys = (object[])BJSON.ToObject(bb); } key = (T)keys[i]; } int offset = Helper.ToInt32(b, idx + 1 + _maxKeySize); int duppage = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4); page.tree.Add(key, new KeyInfo(offset, duppage)); } return page; } else throw new Exception("Page read error header invalid, number = " + number); } } #endregion internal void SavePageList(SafeSortedList _pages, List diskpages) { lock (_fileLock) { T[] keys = _pages.Keys(); int blocknum = 0; // TODO : needed?? //if (_externalStrings) //{ // if (_pagelistalllocblock != null) // _strings.FreeBlocks(_pagelistalllocblock); // blocknum = _strings.SaveData("pagelist", BJSON.ToBJSON(keys, // new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false })); // File.WriteAllBytes(_FileName + ".pagelist", Helper.GetBytes(blocknum, false)); //} // save page list int c = (_pages.Count() / Global.PageItemCount) + 1; // allocate pages needed while (c > diskpages.Count) diskpages.Add(GetNewPageNumber()); byte[] page = new byte[_PageLength]; for (int i = 0; i < (diskpages.Count - 1); i++) { byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]); Buffer.BlockCopy(block, 0, page, 0, block.Length); for (int j = 0; j < Global.PageItemCount; j++) CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page, blocknum); SeekPage(diskpages[i]); _file.Write(page, 0, page.Length); } c = _pages.Count() % Global.PageItemCount; byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1); Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length); int lastoffset = (_pages.Count() / Global.PageItemCount) * Global.PageItemCount; for (int j = 0; j < c; j++) CreatePageListData(_pages, lastoffset, lastblock.Length, j, page, blocknum); SeekPage(diskpages[diskpages.Count - 1]); _file.Write(page, 0, page.Length); } } private void CreatePageListData(SafeSortedList _pages, int offset, int rowindex, int counter, byte[] page, int blocknum) { int idx = rowindex + _rowSize * counter; // key bytes byte[] kk; byte size; if (_externalStrings == false) { kk = _T.GetBytes(_pages.GetKey(counter + offset)); size = (byte)kk.Length; if (size > _maxKeySize) size = _maxKeySize; } else { kk = new byte[4]; Buffer.BlockCopy(Helper.GetBytes(counter + offset, false), 0, kk, 0, 4); size = 4; } // key size = 1 byte page[idx] = size; Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]); // offset = 4 bytes byte[] b = Helper.GetBytes(_pages.GetValue(offset + counter).PageNumber, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length); // add counts b = Helper.GetBytes(_pages.GetValue(offset + counter).UniqueCount, false); Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length); // FEATURE : add dup counts } internal void SaveLastRecordNumber(int recnum) { // save the last record number indexed to the header CreateFileHeader(recnum); } internal void BitmapFlush() { if (_allowDups) _bitmap.Commit(Global.FreeBitmapMemoryOnSave); } } } ================================================ FILE: RaptorDB/Indexes/Indexes.cs ================================================ using System; using System.Collections.Generic; using System.IO; namespace RaptorDB { #region [ TypeIndexes ] internal class TypeIndexes : MGIndex, IIndex where T : IComparable { public TypeIndexes(string path, string filename, byte keysize) : base(path, filename + ".mgidx", keysize, true) { } public void Set(object key, int recnum) { if (key == null) return; // FEATURE : index null values ?? base.Set((T)key, recnum); } public MGRB Query(RDBExpression ex, object from, int maxsize) { T f = default(T); if (typeof(T).Equals(from.GetType()) == false) f = Converter(from); else f = (T)from; return base.Query(ex, f, maxsize); } private T Converter(object from) { if (typeof(T) == typeof(Guid)) { object o = new Guid(from.ToString()); return (T)o; } else return (T)Convert.ChangeType(from, typeof(T)); } void IIndex.FreeMemory() { base.SaveIndex(); base.FreeMemory(); } void IIndex.Shutdown() { base.Shutdown(); } object[] IIndex.GetKeys() { return base.GetKeys(); } public MGRB Query(object fromkey, object tokey, int maxsize) { T f = default(T); if (typeof(T).Equals(fromkey.GetType()) == false) f = (T)Convert.ChangeType(fromkey, typeof(T)); else f = (T)fromkey; T t = default(T); if (typeof(T).Equals(tokey.GetType()) == false) t = (T)Convert.ChangeType(tokey, typeof(T)); else t = (T)tokey; return base.Query(f, t, maxsize); } } #endregion #region [ BoolIndex ] internal class BoolIndex : IIndex { public BoolIndex(string path, string filename, string extension) { // create file _filename = filename + extension; _path = path; if (_path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) _path += Path.DirectorySeparatorChar.ToString(); if (File.Exists(_path + _filename)) ReadFile(); } private MGRB _bits = new MGRB(); private string _filename; private string _path; private object _lock = new object(); public MGRB GetBits() { return _bits.Copy(); } public void Set(object key, int recnum) { lock (_lock) if (key != null) _bits.Set(recnum, (bool)key); } public MGRB Query(RDBExpression ex, object from, int maxsize) { lock (_lock) { bool b = (bool)from; if (b) return _bits; else return _bits.Not(maxsize); } } public void FreeMemory() { lock (_lock) { _bits.Optimize(); SaveIndex(); } } public void Shutdown() { // shutdown WriteFile(); } public void SaveIndex() { WriteFile(); } public void InPlaceOR(MGRB left) { lock (_lock) _bits = _bits.Or(left); } private void WriteFile() { lock (_lock) { _bits.Optimize(); var o = _bits.Serialize(); var b = fastBinaryJSON.BJSON.ToBJSON(o, new fastBinaryJSON.BJSONParameters { UseExtensions = false }); File.WriteAllBytes(_path + _filename, b); } } private void ReadFile() { byte[] b = File.ReadAllBytes(_path + _filename); var o = fastBinaryJSON.BJSON.ToObject(b); _bits = new MGRB(); _bits.Deserialize(o); } public MGRB Query(object fromkey, object tokey, int maxsize) { return Query(RDBExpression.Greater, fromkey, maxsize); // range doesn't make sense here just do from } public object[] GetKeys() { return new object[] { true, false }; } } #endregion #region [ FullTextIndex ] internal class FullTextIndex : Hoot, IIndex { public FullTextIndex(string IndexPath, string FileName, bool docmode, bool sortable, ITokenizer tokenizer) : base(IndexPath, FileName, docmode, tokenizer) { if (sortable) { _idx = new TypeIndexes(IndexPath, FileName, Global.DefaultStringKeySize); _sortable = true; } } private bool _sortable = false; private IIndex _idx; public void Set(object key, int recnum) { base.Index(recnum, (string)key); if (_sortable) _idx.Set(key, recnum); } public MGRB Query(RDBExpression ex, object from, int maxsize) { return base.Query("" + from, maxsize); } public void SaveIndex() { base.Save(); if (_sortable) _idx.SaveIndex(); } public MGRB Query(object fromkey, object tokey, int maxsize) { return base.Query("" + fromkey, maxsize); // range doesn't make sense here just do from } public object[] GetKeys() { if (_sortable) return _idx.GetKeys(); // support get keys else return new object[] { }; } void IIndex.FreeMemory() { base.FreeMemory(); this.SaveIndex(); } void IIndex.Shutdown() { this.SaveIndex(); base.Shutdown(); if (_sortable) _idx.Shutdown(); } } #endregion #region [ EnumIndex ] internal class EnumIndex : MGIndex, IIndex //where T : IComparable { public EnumIndex(string path, string filename) : base(path, filename + ".mgidx", 30, /*Global.PageItemCount,*/ true) { } public void Set(object key, int recnum) { if (key == null) return; // FEATURE : index null values ?? base.Set(key.ToString(), recnum); } public MGRB Query(RDBExpression ex, object from, int maxsize) { T f = default(T); if (typeof(T).Equals(from.GetType()) == false) f = Converter(from); else f = (T)from; return base.Query(ex, f.ToString(), maxsize); } private T Converter(object from) { if (typeof(T) == typeof(Guid)) { object o = new Guid(from.ToString()); return (T)o; } else return (T)Convert.ChangeType(from, typeof(T)); } void IIndex.FreeMemory() { base.SaveIndex(); base.FreeMemory(); } void IIndex.Shutdown() { base.SaveIndex(); base.Shutdown(); } public MGRB Query(object fromkey, object tokey, int maxsize) { T f = default(T); if (typeof(T).Equals(fromkey.GetType()) == false) f = (T)Convert.ChangeType(fromkey, typeof(T)); else f = (T)fromkey; T t = default(T); if (typeof(T).Equals(tokey.GetType()) == false) t = (T)Convert.ChangeType(tokey, typeof(T)); else t = (T)tokey; return base.Query(f.ToString(), t.ToString(), maxsize); } object[] IIndex.GetKeys() { return base.GetKeys(); } } #endregion #region [ NoIndex ] internal class NoIndex : IIndex { public void Set(object key, int recnum) { // ignore set } public MGRB Query(RDBExpression ex, object from, int maxsize) { // always return everything return MGRB.Fill(maxsize); } public void FreeMemory() { } public void Shutdown() { } public void SaveIndex() { } public object[] GetKeys() { return new object[] { }; } public MGRB Query(object fromkey, object tokey, int maxsize) { return MGRB.Fill(maxsize); // TODO : all or none?? } } #endregion } ================================================ FILE: RaptorDB/Indexes/MGIndex.cs ================================================ using System; using System.Collections.Generic; using System.IO; using RaptorDB.Common; namespace RaptorDB { #region [ internal classes ] internal struct PageInfo // FEATURE : change back to class for count access for query caching { public PageInfo(int pagenum, int uniquecount, int duplicatecount) { PageNumber = pagenum; UniqueCount = uniquecount; } public int PageNumber; public int UniqueCount; } internal struct KeyInfo { public KeyInfo(int recnum) { RecordNumber = recnum; DuplicateBitmapNumber = -1; } public KeyInfo(int recnum, int bitmaprec) { RecordNumber = recnum; DuplicateBitmapNumber = bitmaprec; } public int RecordNumber; public int DuplicateBitmapNumber; } internal class Page { public Page() // kludge so the compiler doesn't complain { DiskPageNumber = -1; RightPageNumber = -1; tree = new SafeDictionary(Global.PageItemCount); isDirty = false; FirstKey = default(T); } public int DiskPageNumber; public int RightPageNumber; public T FirstKey; public bool isDirty; public SafeDictionary tree; public List allocblocks = null; // for string keys in HF key store } #endregion internal class MGIndex where T : IComparable { ILog _log = LogManager.GetLogger(typeof(MGIndex)); private SafeSortedList _pageList = new SafeSortedList(); //private SafeDictionary> _cache = new SafeDictionary>(); private IKV> _cache = null;//new SafeSortedList>(); private List _pageListDiskPages = new List(); private IndexFile _index; private bool _AllowDuplicates = true; private int _LastIndexedRecordNumber = 0; public MGIndex(string path, string filename, byte keysize, bool allowdups) { if (Global.UseLessMemoryStructures) _cache = new SafeSortedList>(); else _cache = new SafeDictionary>(); _AllowDuplicates = allowdups; if (path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) path += Path.DirectorySeparatorChar; _index = new IndexFile(path + filename, keysize); // load page list _index.GetPageList(_pageListDiskPages, _pageList, out _LastIndexedRecordNumber); if (_pageList.Count() == 0) { Page page = new Page(); page.FirstKey = (T)RDBDataType.GetEmpty(); page.DiskPageNumber = _index.GetNewPageNumber(); page.isDirty = true; _pageList.Add(page.FirstKey, new PageInfo(page.DiskPageNumber, 0, 0)); _cache.Add(page.DiskPageNumber, page); } } public int GetLastIndexedRecordNumber() { return _LastIndexedRecordNumber; } public MGRB Query(T from, T to, int maxsize) { MGRB bits = new MGRB(); T temp = default(T); if (from.CompareTo(to) > 0) // check values order { temp = from; from = to; to = temp; } // find first page and do > than bool found = false; int startpos = FindPageOrLowerPosition(from, ref found); // find last page and do < than int endpos = FindPageOrLowerPosition(to, ref found); bool samepage = startpos == endpos; // from key page Page page = LoadPage(_pageList.GetValue(startpos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better start position rather than 0 int pos = Array.BinarySearch(keys, from); // FEATURE : rewrite?? if (pos < 0) pos = ~pos; for (int i = pos; i < keys.Length; i++) { T k = keys[i]; int bn = page.tree[k].DuplicateBitmapNumber; if (samepage) { if (k.CompareTo(from) >= 0 && k.CompareTo(to) <= 0) // if from,to same page bits = bits.Or(_index.GetDuplicateBitmap(bn)); } else { if (k.CompareTo(from) >= 0) bits = bits.Or(_index.GetDuplicateBitmap(bn)); } } if (!samepage) { // to key page page = LoadPage(_pageList.GetValue(endpos).PageNumber); keys = page.tree.Keys(); Array.Sort(keys); // find better end position rather than last key pos = Array.BinarySearch(keys, to); if (pos < 0) pos = ~pos; for (int i = 0; i <= pos; i++) { T k = keys[i]; int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(to) <= 0) bits = bits.Or(_index.GetDuplicateBitmap(bn)); } // do all pages in between for (int i = startpos + 1; i < endpos; i++) { doPageOperation(ref bits, i); } } return bits; } public MGRB Query(RDBExpression exp, T from, int maxsize) { T key = from; if (exp == RDBExpression.Equal || exp == RDBExpression.NotEqual) return doEqualOp(exp, key, maxsize); // FEATURE : optimize complement search if page count less for the complement pages if (exp == RDBExpression.Less || exp == RDBExpression.LessEqual) { return doLessOp(exp, key); } else if (exp == RDBExpression.Greater || exp == RDBExpression.GreaterEqual) { return doMoreOp(exp, key); } return new MGRB(); // blank results } private object _setlock = new object(); public void Set(T key, int val) { lock (_setlock) { PageInfo pi; Page page = LoadPage(key, out pi); KeyInfo ki; if (page.tree.TryGetValue(key, out ki)) { // item exists if (_AllowDuplicates) { SaveDuplicate(key, ref ki); // set current record in the bitmap also _index.SetBitmapDuplicate(ki.DuplicateBitmapNumber, val); } ki.RecordNumber = val; page.tree[key] = ki; // structs need resetting } else { // new item ki = new KeyInfo(val); if (_AllowDuplicates) SaveDuplicate(key, ref ki); pi.UniqueCount++; page.tree.Add(key, ki); } if (page.tree.Count() > Global.PageItemCount) SplitPage(page); _LastIndexedRecordNumber = val; page.isDirty = true; } } public bool Get(T key, out int val) { val = -1; PageInfo pi; Page page = LoadPage(key, out pi); KeyInfo ki; bool ret = page.tree.TryGetValue(key, out ki); if (ret) val = ki.RecordNumber; return ret; } public void SaveIndex() { //_log.Debug("Total split time (s) = " + _totalsplits); //_log.Debug("Total pages = " + _pageList.Count); int[] keys = _cache.Keys(); Array.Sort(keys); // save index to disk foreach (var i in keys) { var p = _cache.GetValue(i); if (p.isDirty) { _index.SavePage(p); p.isDirty = false; } } _index.SavePageList(_pageList, _pageListDiskPages); _index.BitmapFlush(); } public void Shutdown() { SaveIndex(); // save page list //_index.SavePageList(_pageList, _pageListDiskPages); // shutdown _index.Shutdown(); } public void FreeMemory() { _index.FreeMemory(); try { List free = new List(); foreach (var k in _cache.Keys()) { var val = _cache.GetValue(k); if (val.isDirty == false) free.Add(k); } _log.Info("releasing page count = " + free.Count + " out of " + _cache.Count()); foreach (var i in free) _cache.Remove(i); } catch { } } public IEnumerable GetDuplicates(T key) { PageInfo pi; Page page = LoadPage(key, out pi); KeyInfo ki; bool ret = page.tree.TryGetValue(key, out ki); if (ret) // get duplicates if (ki.DuplicateBitmapNumber != -1) return _index.GetDuplicatesRecordNumbers(ki.DuplicateBitmapNumber); return new List(); } public void SaveLastRecordNumber(int recnum) { _index.SaveLastRecordNumber(recnum); } public bool RemoveKey(T key) { PageInfo pi; Page page = LoadPage(key, out pi); bool b = page.tree.Remove(key); // TODO : reset the first key for page ?? if (b) { pi.UniqueCount--; // FEATURE : decrease dup count } page.isDirty = true; return b; } #region [ P R I V A T E ] private MGRB doMoreOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); MGRB result = new MGRB(); if (pos < _pageList.Count()) { // all the pages after for (int i = pos + 1; i < _pageList.Count(); i++) doPageOperation(ref result, i); } // key page Page page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better start position rather than 0 pos = Array.BinarySearch(keys, key); if (pos < 0) pos = ~pos; for (int i = pos; i < keys.Length; i++) { T k = keys[i]; int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) > 0) result = result.Or(_index.GetDuplicateBitmap(bn)); if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0) result = result.Or(_index.GetDuplicateBitmap(bn)); } return result; } private MGRB doLessOp(RDBExpression exp, T key) { bool found = false; int pos = FindPageOrLowerPosition(key, ref found); MGRB result = new MGRB(); if (pos > 0) { // all the pages before for (int i = 0; i < pos - 1; i++) doPageOperation(ref result, i); } // key page Page page = LoadPage(_pageList.GetValue(pos).PageNumber); T[] keys = page.tree.Keys(); Array.Sort(keys); // find better end position rather than last key pos = Array.BinarySearch(keys, key); if (pos < 0) pos = ~pos; for (int i = 0; i < pos; i++) { T k = keys[i]; if (k.CompareTo(key) > 0) break; int bn = page.tree[k].DuplicateBitmapNumber; if (k.CompareTo(key) < 0) result = result.Or(_index.GetDuplicateBitmap(bn)); if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0) result = result.Or(_index.GetDuplicateBitmap(bn)); } return result; } private MGRB doEqualOp(RDBExpression exp, T key, int maxsize) { PageInfo pi; Page page = LoadPage(key, out pi); KeyInfo k; if (page.tree.TryGetValue(key, out k)) { int bn = k.DuplicateBitmapNumber; if (exp == RDBExpression.Equal) return _index.GetDuplicateBitmap(bn); else return _index.GetDuplicateBitmap(bn).Not(maxsize); } else { if (exp == RDBExpression.NotEqual) return new MGRB().Not(maxsize); else return new MGRB(); } } private void doPageOperation(ref MGRB res, int pageidx) { Page page = LoadPage(_pageList.GetValue(pageidx).PageNumber); T[] keys = page.tree.Keys(); // avoid sync issues foreach (var k in keys) { int bn = page.tree[k].DuplicateBitmapNumber; res = res.Or(_index.GetDuplicateBitmap(bn)); } } private double _totalsplits = 0; private void SplitPage(Page page) { // split the page DateTime dt = FastDateTime.Now; Page newpage = new Page(); newpage.DiskPageNumber = _index.GetNewPageNumber(); newpage.RightPageNumber = page.RightPageNumber; newpage.isDirty = true; page.RightPageNumber = newpage.DiskPageNumber; _pageList.Remove(page.FirstKey); // get and sort keys T[] keys = page.tree.Keys(); Array.Sort(keys); // copy data to new for (int i = keys.Length / 2; i < keys.Length; i++) { newpage.tree.Add(keys[i], page.tree[keys[i]]); // remove from old page page.tree.Remove(keys[i]); } // set the first key newpage.FirstKey = keys[keys.Length / 2]; // new key // remove keys from page list _pageList.Remove(newpage.FirstKey); page.FirstKey = keys[0]; // new key // re add to page list _pageList.Add(page.FirstKey, new PageInfo(page.DiskPageNumber, page.tree.Count(), 0)); _pageList.Add(newpage.FirstKey, new PageInfo(newpage.DiskPageNumber, newpage.tree.Count(), 0)); _cache.Add(newpage.DiskPageNumber, newpage); _totalsplits += FastDateTime.Now.Subtract(dt).TotalSeconds; } private Page LoadPage(T key, out PageInfo pageinfo) { int pagenum = -1; // find page in list of pages bool found = false; int pos = 0; if (key != null) pos = FindPageOrLowerPosition(key, ref found); pageinfo = _pageList.GetValue(pos); pagenum = pageinfo.PageNumber; Page page; if (_cache.TryGetValue(pagenum, out page) == false) { //load page from disk page = _index.LoadPageFromPageNumber(pagenum); _cache.Add(pagenum, page); } return page; } private Page LoadPage(int pagenum) { Page page; if (_cache.TryGetValue(pagenum, out page) == false) { //load page from disk page = _index.LoadPageFromPageNumber(pagenum); _cache.Add(pagenum, page); } return page; } private void SaveDuplicate(T key, ref KeyInfo ki) { if (ki.DuplicateBitmapNumber == -1) ki.DuplicateBitmapNumber = _index.GetBitmapDuplaicateFreeRecordNumber(); _index.SetBitmapDuplicate(ki.DuplicateBitmapNumber, ki.RecordNumber); } private int FindPageOrLowerPosition(T key, ref bool found) { if (_pageList.Count() == 0) return 0; // binary search int lastlower = 0; int first = 0; int last = _pageList.Count() - 1; int mid = 0; while (first <= last) { mid = (first + last) >> 1; T k = _pageList.GetKey(mid); int compare = k.CompareTo(key); if (compare < 0) { lastlower = mid; first = mid + 1; } if (compare == 0) { found = true; return mid; } if (compare > 0) { last = mid - 1; } } return lastlower; } #endregion internal object[] GetKeys() { List keys = new List(); for (int i = 0; i < _pageList.Count(); i++) { Page page = LoadPage(_pageList.GetValue(i).PageNumber); foreach (var k in page.tree.Keys()) keys.Add(k); } return keys.ToArray(); } internal int Count() { int count = 0; for (int i = 0; i < _pageList.Count(); i++) { Page page = LoadPage(_pageList.GetValue(i).PageNumber); //foreach (var k in page.tree.Keys()) // count++; count += page.tree.Count(); } return count; } } } ================================================ FILE: RaptorDB/Indexes/tokenizer.cs ================================================ using System.Collections.Generic; namespace RaptorDB { class tokenizer : ITokenizer { public Dictionary GenerateWordFreq(string text) { Dictionary dic = new Dictionary(500); char[] chars = text.ToCharArray(); int index = 0; int look = 0; int count = chars.Length; int lastlang = langtype(chars[0]); while (index < count) { int lang = -1; while (look < count) { char c = chars[look]; lang = langtype(c); if (lang == lastlang) look++; else break; } if (lastlang > -1) ParseString(dic, chars, look, index); index = look; lastlang = lang; } return dic; } private static int langtype(char c) { if (char.IsDigit(c)) return 0; else if (char.IsWhiteSpace(c)) return -1; else if (char.IsPunctuation(c)) return -1; else if (char.IsLetter(c)) // FEATURE : language checking here return 1; else return -1; } private static void ParseString(Dictionary dic, char[] chars, int end, int start) { // check if upper lower case mix -> extract words int uppers = 0; bool found = false; for (int i = start; i < end; i++) { if (char.IsUpper(chars[i])) uppers++; } // not all uppercase if (uppers != end - start - 1) { int lastUpper = start; string word = ""; for (int i = start + 1; i < end; i++) { char c = chars[i]; if (char.IsUpper(c)) { found = true; word = new string(chars, lastUpper, i - lastUpper).ToLowerInvariant().Trim(); AddDictionary(dic, word); lastUpper = i; } } if (lastUpper > start) { string last = new string(chars, lastUpper, end - lastUpper).ToLowerInvariant().Trim(); if (word != last) AddDictionary(dic, last); } } if (found == false) { string s = new string(chars, start, end - start).ToLowerInvariant().Trim(); AddDictionary(dic, s); } } private static void AddDictionary(Dictionary dic, string word) { if (word == null) return; int l = word.Length; // too long if (l > Global.DefaultStringKeySize) return; // too short if (l < 2) return; addword(dic, word); } private static void addword(Dictionary dic, string word) { int cc = 0; if (dic.TryGetValue(word, out cc)) dic[word] = ++cc; else dic.Add(word, 1); } } } ================================================ FILE: RaptorDB/Properties/Resources.Designer.cs ================================================ //------------------------------------------------------------------------------ // // This code was generated by a tool. // Runtime Version:4.0.30319.18213 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. // //------------------------------------------------------------------------------ namespace RaptorDB.Properties { using System; /// /// A strongly-typed resource class, for looking up localized strings, etc. /// // This class was auto-generated by the StronglyTypedResourceBuilder // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { private static global::System.Resources.ResourceManager resourceMan; private static global::System.Globalization.CultureInfo resourceCulture; [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] internal Resources() { } /// /// Returns the cached ResourceManager instance used by this class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] internal static global::System.Resources.ResourceManager ResourceManager { get { if (object.ReferenceEquals(resourceMan, null)) { global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("RaptorDB.Properties.Resources", typeof(Resources).Assembly); resourceMan = temp; } return resourceMan; } } /// /// Overrides the current thread's CurrentUICulture property for all /// resource lookups using this strongly typed resource class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] internal static global::System.Globalization.CultureInfo Culture { get { return resourceCulture; } set { resourceCulture = value; } } /// /// Looks up a localized string similar to The following error occurred and the json document is below, you can skip this ///document if you wish by incrementing the %c% file : /// ///%ex% ///------------------------------------------------------------------------------ ///The json document is : /// ///%js% /// ///. /// internal static string msg { get { return ResourceManager.GetString("msg", resourceCulture); } } } } ================================================ FILE: RaptorDB/Properties/Resources.resx ================================================  text/microsoft-resx 2.0 System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 ..\replication\msg.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 ================================================ FILE: RaptorDB/REST/aWebServer.cs ================================================ using System; using System.Net; using System.Text; using System.IO; using System.Threading.Tasks; using System.Reflection; using System.Collections.Generic; using System.IO.Compression; namespace RaptorDB { public abstract class aWebServer { public aWebServer(int HttpPort, bool localonly, AuthenticationSchemes authenticationType, string apiPrefix) { _apiPrefix = apiPrefix; _authenticationType = authenticationType; _localonly = localonly; _port = HttpPort; Console.WriteLine("web port = " + _port); Task.Factory.StartNew(() => Start(), TaskCreationOptions.LongRunning); } public delegate void Handler(HttpListenerContext ctx); public abstract void InitializeCommandHandler(Dictionary handler); #region [ properties ] private string _S = Path.DirectorySeparatorChar.ToString(); internal ILog _log = LogManager.GetLogger(typeof(aWebServer)); private bool _run = true; private int _port; private HttpListener _server; private bool _localonly = false; private Dictionary _handler = new Dictionary(); internal Dictionary _WebCache = new Dictionary(); private AuthenticationSchemes _authenticationType = AuthenticationSchemes.None; private string _apiPrefix = "myapi"; #endregion #region [ web call back handlers ] private void ListenerCallback(IAsyncResult ar) { var listener = ar.AsyncState as HttpListener; var ctx = listener.EndGetContext(ar); _log.Debug("Remote Address = " + ctx.Request.RemoteEndPoint.Address); try { //do some stuff string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower(); if (ctx.User != null) _log.Debug("user : " + ctx.User.Identity.Name); else { //ctx.Response.Redirect("login"); //return; } string webpath = "WEB\\"; webpath = webpath.Replace("\\", "/"); bool handled = false; if (path.StartsWith(_apiPrefix)) { string command = path.Replace(_apiPrefix + "/", ""); if (command.Contains("?")) command = command.Substring(0, command.IndexOf('?') - 1); if (command.Contains("/")) command = command.Substring(0, command.IndexOf('/')); Handler handler = null; if (_handler.TryGetValue(command, out handler)) { handled = true; handler(ctx); } } if (!handled) { if (path == "") { ctx.Response.ContentType = "text/html"; WriteResponse(ctx, 200, ReadFromStream(_WebCache[(webpath + "index.html").ToLower()]), false); } else { if (path.EndsWith(_apiPrefix + ".png") && File.Exists("logo.png")) { OutPutContentType(ctx, path); WriteResponse(ctx, 200, File.ReadAllBytes("logo.png"), false); } else if (_WebCache.ContainsKey((webpath + path).ToLower())) { bool compress = OutPutContentType(ctx, path); var o = _WebCache[(webpath + path).ToLower()]; WriteResponse(ctx, 200, ReadFromStream(o), compress); } else WriteResponse(ctx, 404, "route path not found : " + ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped)); } } ctx.Response.OutputStream.Close(); } catch (Exception ex) { _log.Error(ex); } } #endregion #region [ internal ] internal void Stop() { _run = false; } internal static bool OutPutContentType(HttpListenerContext ctx, string path) { bool compress = false; switch (Path.GetExtension(path).ToLower()) { case ".jpg": case ".jpeg": ctx.Response.ContentType = "image/jpeg"; break; case ".gif": ctx.Response.ContentType = "image/gif"; break; case ".mht": compress = true; ctx.Response.ContentType = "multipart/related"; break; case ".eml": compress = true; ctx.Response.ContentType = "message/rfc822"; break;//"application/x-mimearchive"; break; case ".json": ctx.Response.ContentEncoding = UTF8Encoding.UTF8; ctx.Response.ContentType = "application/json"; break; case ".js": ctx.Response.ContentEncoding = UTF8Encoding.UTF8; ctx.Response.ContentType = "application/javascript"; break; case ".css": ctx.Response.ContentType = "text/css"; break; case ".html": case ".htm": compress = true; ctx.Response.ContentEncoding = UTF8Encoding.UTF8; ctx.Response.ContentType = "text/html"; break; case ".pdf": ctx.Response.ContentType = "application/pdf"; break; case ".doc": case ".docx": ctx.Response.ContentType = "application/msword"; break; case ".xls": case ".xlsx": ctx.Response.ContentType = "application/vnd.ms-excel"; break; default: ctx.Response.ContentType = "application/octet-stream"; break; } return compress; } internal void WriteResponse(HttpListenerContext ctx, int code, string msg) { WriteResponse(ctx, code, Encoding.UTF8.GetBytes(msg), false); } internal void WriteResponse(HttpListenerContext ctx, int code, string msg, bool compress) { WriteResponse(ctx, code, Encoding.UTF8.GetBytes(msg), compress); } internal void WriteResponse(HttpListenerContext ctx, int code, byte[] data, bool compress) { ctx.Response.StatusCode = code; ctx.Response.AppendHeader("Access-Control-Allow-Origin", "*"); byte[] b = data; if (compress == true && b.Length > 100 * 1024) { _log.Debug("original data size : " + b.Length.ToString("#,0")); using (var ms = new MemoryStream()) { using (var zip = new GZipStream(ms, CompressionMode.Compress, true)) zip.Write(b, 0, b.Length); b = ms.ToArray(); } _log.Debug("compressed size : " + b.Length.ToString("#,0")); ctx.Response.AppendHeader("Content-Encoding", "gzip"); } ctx.Response.ContentLength64 = b.LongLength; ctx.Response.OutputStream.Write(b, 0, b.Length); } #endregion #region [ private ] private void Start() { try { InitializeCommandHandler(_handler); ReadResources(); _server = new HttpListener(); if (_authenticationType != AuthenticationSchemes.None) _server.AuthenticationSchemes = _authenticationType; if (_localonly) { _server.Prefixes.Add("http://localhost:" + _port + "/"); _server.Prefixes.Add("http://127.0.0.1:" + _port + "/"); _server.Prefixes.Add("http://" + Environment.MachineName + ":" + _port + "/"); } else _server.Prefixes.Add("http://*:" + _port + "/"); _server.Start(); while (_run) { var context = _server.BeginGetContext(new AsyncCallback(ListenerCallback), _server); context.AsyncWaitHandle.WaitOne(); } } catch (Exception ex) { _log.Error(ex); } } private byte[] ReadFromStream(string name) { using (MemoryStream ms = new MemoryStream()) { Assembly.GetExecutingAssembly().GetManifestResourceStream(name).CopyTo(ms); return ms.ToArray(); } } private void ReadResources() { string name = Assembly.GetExecutingAssembly().GetName().Name + "."; foreach (var r in Assembly.GetExecutingAssembly().GetManifestResourceNames()) { string s = r.Replace(name, ""); if (s.StartsWith("WEB")) { var ext = Path.GetExtension(s); s = s.Replace(ext, "").Replace(".", "/"); var p = s + ext; _WebCache.Add(p.ToLower(), r); } } } #endregion } } ================================================ FILE: RaptorDB/REST/rdbRest.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Net; using System.Reflection; using System.Text; using System.Text.RegularExpressions; namespace RaptorDB { #region [ rdb rest helper classes ] public class RDBRoute { public string URL; public Type EntityType; public string Viewname; public ServerSideFunc function; public override string ToString() { if (EntityType != null) return "POST-able type : " + EntityType.Name; else if (function != null) return "Server side function"; else if (Viewname != "") return "View name : " + Viewname; return "Undefined"; } } public interface IRouteAPI { void AddRoute(RDBRoute route); void RegisterView(View view); } public interface IRDBRouting { void Initialize(IRouteAPI api); } public class RDBJsonContainer { public string URL; public DateTime date; public string json; public string useragent; public Guid docid; } #endregion class rdbRest : aWebServer, IRouteAPI { public rdbRest(int HttpPort, RaptorDB rdb, string routingpath, bool localonly) : base(HttpPort, localonly, System.Net.AuthenticationSchemes.None, "raptordb") { _rdb = rdb; //save = _rdb.GetType().GetMethod("Save", BindingFlags.Instance | BindingFlags.Public); //register = _rdb.GetType().GetMethod("RegisterView", BindingFlags.Instance | BindingFlags.Public); if (_S == "/") _path = routingpath.Replace("\\", "/"); else _path = routingpath; } private string _S = Path.DirectorySeparatorChar.ToString(); private RaptorDB _rdb; private string _path; private SafeDictionary _routing = new SafeDictionary(); //private KeyStore _jsonstore; public new void Stop() { base.Stop(); //_jsonstore.Shutdown(); //_rdb.Shutdown(); } public void AddRoute(RDBRoute route) { _log.Debug("adding route : " + route.URL); _routing.Add(route.URL.ToLower(), route); } public void RegisterView(View view) { _log.Debug("registering view : " + view.Name); AddRoute(new RDBRoute { URL = "RaptorDB/Views/" + view.Name, Viewname = view.Name }); _rdb.RegisterView(view); } public override void InitializeCommandHandler(Dictionary _handler) { _handler.Add("getroutes", (ctx) => { List o = new List(); foreach (var rr in _routing) o.Add(new { URL = rr.Value.URL, Description = rr.Value.ToString() }); OutputJsonData(ctx, o); }); _handler.Add("getviews", (ctx) => { List o = new List(); foreach (var v in _rdb.GetViews()) o.Add(new { Name = v.Name, Description = v.Description, BackgroundIndexing = v.BackgroundIndexing, Version = v.Version, isPrimaryList = v.isPrimaryList }); OutputJsonData(ctx, o); }); _handler.Add("getschema", (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); if (qry == "") { WriteResponse(ctx, 404, "GetSchema requires a viewname to be defined e.g. ?view=customerview"); } else { List o = new List(); string view = qry.Split('=')[1]; var sc = _rdb.GetSchema(view); foreach (var i in sc.Columns) o.Add(new { ColumnName = i.Key, Type = i.Value.Name }); OutputJsonData(ctx, o); } }); _handler.Add("systeminfo", (ctx) => { var oo = GetInfo(); var s = fastJSON.JSON.ToJSON(oo, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("action", (ctx) => { string action = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); switch (action) { case "backup": _rdb.Backup(); WriteResponse(ctx, 200, "\"Done\""); break; case "compact": _rdb.GetKVHF().CompactStorageHF(); WriteResponse(ctx, 200, "\"Done\""); break; case "getconfigs": WriteResponse(ctx, 200, File.ReadAllText(_path + "raptordb.config")); break; } }); _handler.Add("docget", // takes : guid (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var g = Guid.Parse(qry); _log.Debug("docid = " + qry); var s = fastJSON.JSON.ToNiceJSON(_rdb.Fetch(g), new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("dochistory", // takes : guid (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var g = Guid.Parse(qry); var h = _rdb.FetchHistoryInfo(g); _log.Debug("docid = " + qry); var s = fastJSON.JSON.ToJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("docversion", // takes : version (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var v = int.Parse(qry); var oo = _rdb.FetchVersion(v); var s = fastJSON.JSON.ToNiceJSON(oo, new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("fileget", // takes : guid (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var g = Guid.Parse(qry); _log.Debug("fileid = " + qry); var s = fastJSON.JSON.ToNiceJSON(_rdb.FetchBytes(g), new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("filehistory", // takes : guid (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var g = Guid.Parse(qry); var h = _rdb.FetchBytesHistoryInfo(g); _log.Debug("fileid = " + qry); var s = fastJSON.JSON.ToJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("fileversion", // takes : version (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var v = int.Parse(qry); var oo = _rdb.FetchBytesVersion(v); var s = fastJSON.JSON.ToNiceJSON(oo, new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("docsearch", // takes : string & count =x &start=y (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); int start = 0; int count = -1; var m = _start_regex.Match(qry); if (m.Success) { start = int.Parse(m.Groups["start"].Value); qry = qry.Replace(m.Value, ""); } m = _count_regex.Match(qry); if (m.Success) { count = int.Parse(m.Groups["count"].Value); qry = qry.Replace(m.Value, ""); } var h = _rdb.FullTextSearch(qry); List list = new List(); _log.Debug("search = " + qry); if (count > -1 && h.Length > 0) { int c = h.Length; for (int i = start; i < start + count && i < c; i++) list.Add(h[i]); } var obj = new { Items = list, Count = count, TotalCount = h.Length }; var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("hfkeys", // takes : count =x &start=y (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); int start = 0; int count = -1; var m = _start_regex.Match(qry); if (m.Success) { start = int.Parse(m.Groups["start"].Value); qry = qry.Replace(m.Value, ""); } m = _count_regex.Match(qry); if (m.Success) { count = int.Parse(m.Groups["count"].Value); qry = qry.Replace(m.Value, ""); } var h = _rdb.GetKVHF().GetKeysHF(); List list = new List(); if (count > -1 && h.Length > 0) { int c = h.Length; for (int i = start; i < start + count && i < c; i++) list.Add(h[i]); } var obj = new { Items = list, Count = count, TotalCount = h.Length }; var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("hfget", // takes : string (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); var h = _rdb.GetKVHF().GetObjectHF(qry); var s = fastJSON.JSON.ToNiceJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); }); _handler.Add("viewinfo", // takes : viewname (ctx) => { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); if (qry == "") { WriteResponse(ctx, 404, "ViewInfo requires a viewname to be defined e.g. ?customerview"); } else { var vi = GetViewInfo(qry); if (vi == "") WriteResponse(ctx, 500, "View not found."); else { ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, vi); } } }); _handler.Add("excelexport", (ctx) => { string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower(); var data = DoQuery(_rdb, ctx, path.Replace("raptordb/excelexport/", ""), null); ctx.Response.AddHeader("content-disposition", "attachment;filename=" + data.Title + ".csv"); ctx.Response.AddHeader("Content-Type", "application/vnd.ms-excel"); _log.Debug("exporting to excel rows : " + data.Rows.Count); WriteResponse(ctx, 200, WriteCsv(data.Rows), true); }); _handler.Add("views", (ctx) => { string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower(); ProcessGET(_rdb, ctx, path.Replace("raptordb/views/", ""), null); }); } #region [ private ] private string GetViewInfo(string name) { var v = _rdb.GetViews().Find(x => x.Name.ToLower() == name.ToLower()); if (v == null) return ""; var p = new Dictionary(); foreach (var pr in v.Schema.GetProperties()) p.Add(pr.Name, pr.PropertyType.ToString()); foreach (var pr in v.Schema.GetFields()) p.Add(pr.Name, pr.FieldType.ToString()); var obj = new { View = v, Schema = p }; var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true }); return s; } private object GetInfo() { var ts = _rdb.Uptime(); var s = "" + ts.Days + " days, " + ts.Hours + " hours, " + ts.Minutes + " mins, " + ts.Seconds + " secs"; // get info here return new { DocumentCount = _rdb.DocumentCount(), FileCount = _rdb.FileCount(), OSVersion = Environment.OSVersion.ToString(), NumberOfViews = _rdb.GetViews().Count, HighFrequncyItems = _rdb.GetKVHF().CountHF(), RaptorDBVersion = FileVersionInfo.GetVersionInfo(this.GetType().Assembly.Location).ProductVersion.ToString(), DataFolderSize = _rdb.GetDataFolderSize(), Uptime = s, MemoryUsage = GetMemoryUsage(), LogItems = LogManager.GetLastLogs() }; } private string GetMemoryUsage() // KLUDGE but works { try { string fname = Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location); ProcessStartInfo ps = new ProcessStartInfo("tasklist"); ps.Arguments = "/fi \"IMAGENAME eq " + fname + ".*\" /FO CSV /NH"; ps.RedirectStandardOutput = true; ps.CreateNoWindow = true; ps.UseShellExecute = false; var p = Process.Start(ps); if (p.WaitForExit(1000)) { var s = p.StandardOutput.ReadToEnd().Split('\"'); return s[9].Replace("\"", ""); } } catch { } return "Unable to get memory usage"; } private string WriteCsv(List data)//, Stream stream) { StringBuilder output = new StringBuilder(); var o = data[0]; foreach (var prop in o.GetType().GetProperties()) { output.Append(prop.Name); // header output.Append(","); } foreach (var prop in o.GetType().GetFields()) { output.Append(prop.Name); // header output.Append(","); } output.AppendLine(); foreach (var item in data) { foreach (var prop in o.GetType().GetProperties()) { output.Append("\"" + prop.GetValue(item, null)); output.Append("\","); } foreach (var prop in o.GetType().GetFields()) { output.Append("\"" + prop.GetValue(item)); output.Append("\","); } output.AppendLine(); } return output.ToString(); } //private MethodInfo register = null; //private void CompileAndRegisterScriptRoutes(string routefolder) //{ // // compile & register views // string[] files = Directory.GetFiles(routefolder, "*.route"); // foreach (var fn in files) // { // Assembly a = CompileScript(fn); // if (a != null) // { // foreach (var t in a.GetTypes()) // { // if (typeof(IRDBRouting).IsAssignableFrom(t)) // { // IRDBRouting r = (IRDBRouting)Activator.CreateInstance(t); // r.Initialize(this); // } // // load views if exists // foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false)) // { // try // { // object o = Activator.CreateInstance(t); // // handle types when view also // Type[] args = t.GetGenericArguments(); // if (args.Length == 0) // args = t.BaseType.GetGenericArguments(); // Type tt = args[0]; // var m = register.MakeGenericMethod(new Type[] { tt }); // m.Invoke(_rdb, new object[] { o }); // } // catch (Exception ex) // { // _log.Error(ex); // } // } // } // } // } //} //private Assembly CompileScript(string file) //{ // try // { // _log.Debug("Compiling route script : " + file); // CodeDomProvider compiler = CodeDomProvider.CreateProvider("CSharp"); // CompilerParameters compilerparams = new CompilerParameters(); // compilerparams.GenerateInMemory = false; // compilerparams.GenerateExecutable = false; // compilerparams.OutputAssembly = file.Replace(".route", ".dll"); // compilerparams.CompilerOptions = "/optimize"; // Regex regex = new Regex( // @"\/\/\s*ref\s*\:\s*(?.*)", // System.Text.RegularExpressions.RegexOptions.IgnoreCase); // compilerparams.ReferencedAssemblies.Add(typeof(View<>).Assembly.Location); //raprotdb.common.dll // compilerparams.ReferencedAssemblies.Add(typeof(object).Assembly.Location); //mscorlib.dll // compilerparams.ReferencedAssemblies.Add(typeof(System.Uri).Assembly.Location); //system.dll // compilerparams.ReferencedAssemblies.Add(typeof(System.Linq.Enumerable).Assembly.Location);//system.core.dll // compilerparams.ReferencedAssemblies.Add(typeof(IRDBRouting).Assembly.Location); //raptordb.rest.dll // foreach (Match m in regex.Matches(File.ReadAllText(file))) // { // string str = m.Groups["refs"].Value.Trim(); // Assembly a = Assembly.LoadWithPartialName(Path.GetFileNameWithoutExtension(str));//load from GAC if possible // if (a != null) // compilerparams.ReferencedAssemblies.Add(a.Location); // else // { // string assm = Path.GetDirectoryName(this.GetType().Assembly.Location) + _S + str; // a = Assembly.LoadFrom(assm); // if (a != null) // compilerparams.ReferencedAssemblies.Add(a.Location); // else // _log.Error("unable to find referenced file for view compiling : " + str); // } // } // CompilerResults results = compiler.CompileAssemblyFromFile(compilerparams, file); // if (results.Errors.HasErrors == true) // { // _log.Error("Error compiling route definition : " + file); // foreach (var e in results.Errors) // _log.Error(e.ToString()); // return null; // } // return results.CompiledAssembly; // } // catch (Exception ex) // { // _log.Error("Error compiling route definition : " + file); // _log.Error(ex); // return null; // } //} private void OutputJsonData(HttpListenerContext ctx, List o) { Result resf = new Result(true); resf.Rows = o; resf.TotalCount = o.Count; resf.Count = o.Count; var s = fastJSON.JSON.ToJSON(resf, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); } private Regex _start_regex = new Regex(@"[\?\&]?\s*start\s*\=\s*[-+]?(?\d*)", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled); private Regex _count_regex = new Regex(@"[\?\&]?\s*count\s*\=\s*[-+]?(?\d*)", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled); private Regex _order_regex = new Regex(@"[\?\&]?\s*orderby\s*\=\s*[-+]?(?.*)", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled); private Result DoQuery(IRaptorDB rdb, HttpListenerContext ctx, string path, RDBRoute route) { string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped); string viewname = path; if (route != null) { //if (route.EntityType != null) //{ // if (qry != "") // { // // fetch the json document // string[] s = qry.Split('='); // object obj = null; // if (_jsonstore.GetObject(Guid.Parse(s[1].Replace("\"", "")), out obj)) // { // RDBJsonContainer d = (RDBJsonContainer)obj; // WriteResponse(ctx, 200, d.json); // return; // } // } // WriteResponse(ctx, 404, "GUID not found :" + qry); // return; //} if (route.Viewname == null && route.function != null) { viewname = route.Viewname; var o = route.function(_rdb, qry); Result resf = new Result(true); resf.Rows = o; resf.TotalCount = o.Count; resf.Count = o.Count; resf.Title = route.Viewname; return resf; } } // parse "start" and "count" from qry if exists int start = 0; int count = -1; string orderby = ""; var m = _start_regex.Match(qry); if (m.Success) { start = int.Parse(m.Groups["start"].Value); qry = qry.Replace(m.Value, ""); } m = _count_regex.Match(qry); if (m.Success) { count = int.Parse(m.Groups["count"].Value); qry = qry.Replace(m.Value, ""); } m = _order_regex.Match(qry); if (m.Success) { orderby = m.Groups["orderby"].Value; qry = qry.Replace(m.Value, ""); } var res = rdb.Query(viewname, qry, start, count, orderby); res.Title = viewname; return res; } private void ProcessGET(IRaptorDB rdb, HttpListenerContext ctx, string path, RDBRoute route) { try { var result = DoQuery(rdb, ctx, path, route); var s = fastJSON.JSON.ToJSON(result, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true }); ctx.Response.ContentType = "application/json"; WriteResponse(ctx, 200, s); return; } catch (Exception ex) { WriteResponse(ctx, 500, "" + ex); } } #endregion } } ================================================ FILE: RaptorDB/RaptorDB.cs ================================================ using RaptorDB.Common; using RaptorDB.Views; using System; using System.CodeDom.Compiler; using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.IO.Compression; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Text; using System.Text.RegularExpressions; using System.Threading; // ----- Feature list ------- // TODO : enum in row schema support // TODO : validate view schema with mapper on startup ?? // TODO : HFKV transaction mode set and rollback handling // TODO : put page list string key alloc block num in index file header namespace RaptorDB { public class RaptorDB : IRaptorDB { private RaptorDB(string FolderPath, ITokenizer tokenizer) { if (tokenizer != null) _tokenizer = tokenizer; else _tokenizer = new tokenizer(); // speed settings fastJSON.JSON.Parameters.ParametricConstructorOverride = true; fastBinaryJSON.BJSON.Parameters.ParametricConstructorOverride = true; fastJSON.JSON.Parameters.UseEscapedUnicode = false; // backward compatibility //fastBinaryJSON.BJSON.Parameters.v1_4TypedArray = true; fastJSON.Reflection.RDBMode = true; if (_S == "/") FolderPath = FolderPath.Replace("\\", "/"); // create folders Directory.CreateDirectory(FolderPath); string foldername = Path.GetFullPath(FolderPath); if (foldername.EndsWith(_S) == false) foldername += _S; _Path = foldername; // if configs !exists create template config files CreateTemplateConfigFiles(); Initialize(); } public static RaptorDB Open(string FolderPath) { return new RaptorDB(FolderPath, new tokenizer()); } public static RaptorDB Open(string FolderPath, ITokenizer tokenizer) { return new RaptorDB(FolderPath, tokenizer); } private string _S = Path.DirectorySeparatorChar.ToString(); private ILog _log = LogManager.GetLogger(typeof(RaptorDB)); private ViewManager _viewManager; private KeyStore _objStore; private KeyStore _fileStore; private KeyStoreHF _objHF; private string _Path = ""; private int _LastRecordNumberProcessed = -1; // used by background saver private int _LastFulltextIndexed = -1; // used by the fulltext indexer private int _LastBackupRecordNumber = -1; private int _CurrentRecordNumber = -1; private System.Timers.Timer _saveTimer; private System.Timers.Timer _fulltextTimer; private System.Timers.Timer _freeMemTimer; private System.Timers.Timer _processinboxTimer; private bool _shuttingdown = false; private bool _pauseindexer = false; private MethodInfo otherviews = null; private MethodInfo save = null; private MethodInfo saverep = null; private SafeDictionary _savecache = new SafeDictionary(); private SafeDictionary _saverepcache = new SafeDictionary(); private FullTextIndex _fulltextindex; private CronDaemon _cron; private Replication.ReplicationServer _repserver; private Replication.ReplicationClient _repclient; private DateTime _startTime = DateTime.Now; private ITokenizer _tokenizer = new tokenizer(); private int _RaptorDBVersion = 4; #region [ P U B L I C I N T E R F A C E ] /// /// Save files to RaptorDB /// /// /// public bool SaveBytes(Guid docID, byte[] bytes) { // save files in storage _fileStore.SetBytes(docID, bytes); return true; } /// /// Delete a document (note data is not lost just flagged as deleted) /// /// /// public bool Delete(Guid docid) { bool b = _objStore.Delete(docid); _viewManager.Delete(docid); return b; } /// /// Delete a file (note data is not lost just flagged as deleted) /// /// /// public bool DeleteBytes(Guid bytesid) { return _fileStore.Delete(bytesid); } /// /// Save a document /// /// /// /// /// public bool Save(Guid docid, T data) { string viewname = _viewManager.GetPrimaryViewForType(data.GetType()); if (viewname == "" && Global.RequirePrimaryView == true) { _log.Warn("Primary View not defined for object : " + data.GetType()); return false; } _pauseindexer = true; if (viewname != "" && _viewManager.isTransaction(viewname)) { _log.Debug("TRANSACTION started for docid : " + docid); // add code here _viewManager.StartTransaction(); bool b = SaveInPrimaryViewTransaction(viewname, docid, data); if (b == true) { b = SaveToConsistentViewsTransaction(docid, data); if (b == true) { b = SaveInOtherViewsTransaction(docid, data); if (b == true) { _viewManager.Commit(Thread.CurrentThread.ManagedThreadId); int recnum = _objStore.SetObject(docid, data); _CurrentRecordNumber = recnum; _pauseindexer = false; return true; } } } _viewManager.Rollback(Thread.CurrentThread.ManagedThreadId); _pauseindexer = false; return false; } else { int recnum = _objStore.SetObject(docid, data); _CurrentRecordNumber = recnum; if (viewname != "") { SaveInPrimaryView(viewname, docid, data); SaveToConsistentViews(docid, data); if (Global.BackgroundSaveToOtherViews == false) { SaveInOtherViews(docid, data); _LastRecordNumberProcessed = recnum; } } _pauseindexer = false; return true; } } internal TimeSpan Uptime() { return DateTime.Now.Subtract(_startTime); } internal object FileCount() { return _fileStore.Count(); } /// /// Query any view -> get all rows /// /// /// /// public Result Query(string viewname) { return _viewManager.Query(viewname, 0, -1); } /// /// Query a view using a string filter /// /// /// /// public Result Query(string viewname, string filter) { if (filter == "") return _viewManager.Query(viewname, 0, -1); return _viewManager.Query(viewname, filter, 0, -1); } /// /// Fetch a document by it's ID /// /// /// public object Fetch(Guid docID) { object b = null; _objStore.GetObject(docID, out b); return b; } /// /// Fetch file data by it's ID /// /// /// public byte[] FetchBytes(Guid fileID) { byte[] b = null; if (_fileStore.GetBytes(fileID, out b)) return b; else return null; } /// /// Register a view /// /// /// public void RegisterView(View view) { _viewManager.RegisterView(view); } /// /// Shutdown the database engine and flush memory to disk /// public void Shutdown() { if (_shuttingdown == true) return; _shuttingdown = true; if (_restServer != null) _restServer.Stop(); _processinboxTimer.Enabled = false; _saveTimer.Enabled = false; _freeMemTimer.Enabled = false; _fulltextTimer.Enabled = false; if (_repserver != null) _repserver.Shutdown(); if (_repclient != null) _repclient.Shutdown(); // feature : write global or something else? File.WriteAllText(_Path + "RaptorDB.config", fastJSON.JSON.ToNiceJSON(new Global(), new fastJSON.JSONParameters { UseExtensions = false })); if (_cron != null) _cron.Stop(); _fulltextTimer.Stop(); _fulltextindex.Shutdown(); _log.Debug("Shutting down"); _saveTimer.Stop(); _viewManager.ShutDown(); _objStore.Shutdown(); _fileStore.Shutdown(); _objHF.Shutdown(); // save records _log.Debug("last full text record = " + _LastFulltextIndexed); File.WriteAllBytes(_Path + "Data" + _S + "Fulltext" + _S + "_fulltext.rec", Helper.GetBytes(_LastFulltextIndexed, false)); _log.Debug("last record = " + _LastRecordNumberProcessed); File.WriteAllBytes(_Path + "Data" + _S + "_lastrecord.rec", Helper.GetBytes(_LastRecordNumberProcessed, false)); _log.Debug("last backup record = " + _LastBackupRecordNumber); File.WriteAllBytes(_Path + "Backup" + _S + "LastBackupRecord.rec", Helper.GetBytes(_LastBackupRecordNumber, false)); _log.Debug("Shutting down log."); _log.Debug("RaptorDB done."); LogManager.Shutdown(); } #region [ BACKUP/RESTORE and REPLICATION ] private object _backuplock = new object(); /// /// Backup the document storage file incrementally to "Backup" folder /// /// True = done public bool Backup() { if (_LastBackupRecordNumber >= _CurrentRecordNumber) return false; lock (_backuplock) { _log.Debug("Backup Started..."); string tempp = _Path + "Temp" + _S + DateTime.Now.ToString("yyyy-MM-dd-HH-mm"); Directory.CreateDirectory(tempp); StorageFile backup = new StorageFile(tempp + _S + "backup.mgdat", SF_FORMAT.BSON, true); _log.Debug("Copying data to backup"); if (_LastBackupRecordNumber == -1) _LastBackupRecordNumber = 0; int rec = _objStore.CopyTo(backup, _LastBackupRecordNumber); backup.Shutdown(); _log.Debug("Last backup rec# = " + rec); // compress the file using (FileStream read = File.OpenRead(tempp + _S + "backup.mgdat")) using (FileStream outp = File.Create(tempp + _S + "backup.mgdat.gz")) CompressForBackup(read, outp); _log.Debug("Backup compressed and done"); File.Move(tempp + _S + "backup.mgdat.gz", _Path + "Backup" + _S + DateTime.Now.ToString("yyyy-MM-dd-HH-mm") + ".mgdat.gz"); _log.Debug("last backup record = " + _LastBackupRecordNumber); File.WriteAllBytes(_Path + "Backup" + _S + "LastBackupRecord.rec", Helper.GetBytes(_LastBackupRecordNumber, false)); // cleanup temp folder Directory.Delete(tempp, true); _log.Debug("Backup done."); _LastBackupRecordNumber = rec; return true; } } private DateTime _lastFailedTime = DateTime.Now; private object _replock = new object(); private void ProcessReplicationInbox(string inboxfolder) { lock (_replock) { if (Directory.Exists(inboxfolder) == false) return; string[] files = Directory.GetFiles(inboxfolder, "*.counter"); // check if ".counter" file exists if (files.Length > 0) { // FEATURE: if lastfailedtime < 15 -> wait 15 min and retry (avoid extra cpu burning) // recovery mode string fn = files[0]; int start = -1; if (int.TryParse(File.ReadAllText(fn).Trim(), out start)) { if (DoRepProcessing(fn.Replace(".counter", ".mgdat"), start) == false) return; } else { _log.Error("Unable to parse counter value in : " + fn); return; } } files = Directory.GetFiles(inboxfolder, "*.gz"); Array.Sort(files); foreach (var filename in files) { string tmp = filename.Replace(".gz", "");// FEATURE : to temp folder ?? if (File.Exists(tmp)) File.Delete(tmp); using (FileStream read = File.OpenRead(filename)) using (FileStream outp = File.Create(tmp)) DecompressForRestore(read, outp); _log.Debug("Uncompress done : " + Path.GetFileName(tmp)); if (DoRepProcessing(tmp, 0) == false) return; if (_shuttingdown) return; } } } private bool DoRepProcessing(string filename, int start) { string fn = Path.GetFileNameWithoutExtension(filename); string path = Path.GetDirectoryName(filename); StorageFile sf = StorageFile.ReadForward(filename); int counter = 0; if (start > 0) _log.Debug("skipping replication items : " + start); foreach (var i in sf.ReadOnlyEnumerate()) { if (start > 0) // skip already done { start--; counter++; } else { if (i.meta.isDeleted) DeleteReplicate(i.meta.key); else { try { object obj = CreateObject(i.data); var m = GetSaveReplicate(obj.GetType()); m.Invoke(this, new object[] { i.meta.key, obj }); } catch (Exception ex) { _log.Error(ex); sf.Shutdown(); string err = Properties.Resources.msg.Replace("%js%", fastJSON.JSON.Beautify(Helper.GetString(i.data, 0, (short)i.data.Length))) .Replace("%ex%", "" + ex) .Replace("%c%", path + _S + fn + ".counter"); File.WriteAllText(path + _S + fn + ".error.txt", err); _lastFailedTime = DateTime.Now; return false; } } counter++; File.WriteAllText(path + _S + fn + ".counter", "" + counter); if (_shuttingdown) { _log.Debug("shutting down before replicate data completed..."); sf.Shutdown(); return false; } } } sf.Shutdown(); _log.Debug("File replicate complete : " + Path.GetFileName(filename)); foreach (var f in Directory.GetFiles(path, fn + ".*")) File.Delete(f); return true; } private void DeleteReplicate(Guid docid) { bool b = _objStore.DeleteReplicated(docid); _viewManager.Delete(docid); } private object _restoreLock = new object(); /// /// Start background restore of backups in the "Restore" folder /// public void Restore() { lock (_restoreLock) { try { string[] files = Directory.GetFiles(_Path + "Restore", "*.counter"); // check if ".counter" file exists if (files.Length > 0) { // resume mode string fn = files[0]; int start = -1; if (int.TryParse(File.ReadAllText(fn).Trim(), out start)) { if (DoRestoreProcessinng(fn.Replace(".counter", ".mgdat"), start) == false) return; } else { _log.Error("Unable to parse counter value in : " + fn); return; } } // do restore files = Directory.GetFiles(_Path + "Restore", "*.gz"); Array.Sort(files); _log.Debug("Restoring file count = " + files.Length); foreach (string file in files) { string tmp = file.Replace(".gz", "");// FEATURE : to temp folder ?? if (File.Exists(tmp)) File.Delete(tmp); using (FileStream read = File.OpenRead(file)) using (FileStream outp = File.Create(tmp)) DecompressForRestore(read, outp); _log.Debug("Uncompress done : " + Path.GetFileName(tmp)); if (DoRestoreProcessinng(tmp, 0)) File.Move(file, _Path + "Restore" + _S + "Done" + _S + Path.GetFileName(file)); } } catch (Exception ex) { _log.Error(ex); } } } private bool DoRestoreProcessinng(string filename, int start) { string fn = Path.GetFileNameWithoutExtension(filename); string path = Path.GetDirectoryName(filename); int counter = 0; StorageFile sf = StorageFile.ReadForward(filename); foreach (var i in sf.ReadOnlyEnumerate()) { if (start > 0) { start--; counter++; } else { if (i.meta.isDeleted) Delete(i.meta.key); else { object obj = CreateObject(i.data); var m = GetSave(obj.GetType()); m.Invoke(this, new object[] { i.meta.key, obj }); } counter++; File.WriteAllText(path + _S + fn + ".counter", "" + counter); if (_shuttingdown) { _log.Debug("shutting down before restore completed..."); sf.Shutdown(); return false; } } } sf.Shutdown(); _log.Debug("File restore complete : " + Path.GetFileName(filename)); foreach (var f in Directory.GetFiles(path, fn + ".*")) File.Delete(f); return true; } private bool SaveReplicationObject(Guid docid, T data) { string viewname = _viewManager.GetPrimaryViewForType(data.GetType()); if (viewname == "") { _log.Debug("Primary View not defined for object : " + data.GetType()); return false; } _pauseindexer = true; int recnum = _objStore.SetReplicationObject(docid, data); _CurrentRecordNumber = recnum; SaveInPrimaryView(viewname, docid, data); SaveToConsistentViews(docid, data); if (Global.BackgroundSaveToOtherViews == false) { SaveInOtherViews(docid, data); _LastRecordNumberProcessed = recnum; } _pauseindexer = false; return true; } #endregion /// /// Add a user (only supported in server mode) /// /// /// /// /// public bool AddUser(string username, string oldpassword, string newpassword) { return false; } /// /// Execute a server side string filter query /// /// /// /// public object[] ServerSide(ServerSideFunc func, string filter) { return func(this, filter).ToArray(); } /// /// Execute a server side LINQ query /// /// /// /// /// public object[] ServerSide(ServerSideFunc func, Expression> filter) { LINQString ls = new LINQString(); ls.Visit(filter); return func(this, ls.sb.ToString()).ToArray(); } /// /// Full text search the entire original document /// /// /// public int[] FullTextSearch(string filter) { var wbmp = _fulltextindex.Query(filter, _objStore.RecordCount()); List a = new List(); a.AddRange(wbmp.GetBitIndexes()); return a.ToArray(); } /// /// Query a view /// /// /// /// public Result Query(Expression> filter) { return _viewManager.Query(filter, 0, -1); } /// /// Query a view with paging /// /// /// /// /// /// public Result Query(Expression> filter, int start, int count) { return _viewManager.Query(filter, start, count, ""); } /// /// Query a view with paging and order by /// /// /// /// /// /// /// public Result Query(Expression> filter, int start, int count, string orderby) { return _viewManager.Query(filter, start, count, orderby); } /// /// Query a view /// /// /// /// public Result Query(string filter) { return _viewManager.Query(filter, 0, -1); } /// /// Query a view with paging /// /// /// /// /// /// public Result Query(string filter, int start, int count) { return _viewManager.Query(filter, start, count); } /// /// Count with filter /// /// /// /// public int Count(Expression> filter) { return _viewManager.Count(filter); } /// /// /// /// /// /// /// public Result Query(string viewname, int start, int count) { return _viewManager.Query(viewname, start, count); } /// /// /// /// /// /// /// /// public Result Query(string viewname, string filter, int start, int count) { return _viewManager.Query(viewname, filter, start, count); } /// /// Count all data associated with View name /// /// /// public int Count(string viewname) { return _viewManager.Count(viewname, ""); } /// /// Count all data associated with View name and string filter /// /// /// /// public int Count(string viewname, string filter) { return _viewManager.Count(viewname, filter); } /// /// Fetch the change history for a document /// /// /// public int[] FetchHistory(Guid docid) { return _objStore.GetHistory(docid); } /// /// Fetch a change history for a file /// /// /// public int[] FetchBytesHistory(Guid fileid) { return _fileStore.GetHistory(fileid); } /// /// Fetch the specific document version /// /// /// public object FetchVersion(int versionNumber) { StorageItem meta = null; return _objStore.GetObject(versionNumber, out meta); } /// /// Fetch the specific file version /// /// /// public byte[] FetchBytesVersion(int versionNumber) { StorageItem meta = null; return _fileStore.GetBytes(versionNumber, out meta); } /// /// Get the current registered views /// /// public List GetViews() { return _viewManager.GetViews(); } /// /// Get the schema for a view /// /// /// public ViewRowDefinition GetSchema(string view) { return _viewManager.GetSchema(view); } /// /// Query a view with paging and ordering /// /// /// /// /// /// /// public Result Query(string viewname, string filter, int start, int count, string orderby) { return _viewManager.Query(viewname, filter, start, count, orderby); } /// /// Query a view with paging and ordering /// /// /// /// /// /// /// public Result Query(string filter, int start, int count, string orderby) { return _viewManager.Query(filter, start, count, orderby); } /// /// Get the history information for a document /// /// /// public HistoryInfo[] FetchHistoryInfo(Guid docid) { List h = new List(); foreach (int i in FetchHistory(docid)) { HistoryInfo hi = new HistoryInfo(); hi.Version = i; var o = _objStore.GetMeta(i); hi.ChangeDate = o.date; if (o.isDeleted == false) h.Add(hi); } return h.ToArray(); } /// /// Get the history information for a file /// /// /// public HistoryInfo[] FetchBytesHistoryInfo(Guid docid) { List h = new List(); foreach (int i in FetchBytesHistory(docid)) { HistoryInfo hi = new HistoryInfo(); hi.Version = i; var o = _fileStore.GetMeta(i); hi.ChangeDate = o.date; if (o.isDeleted == false) h.Add(hi); } return h.ToArray(); } /// /// Direct delete from a view /// /// /// /// public int ViewDelete(Expression> filter) { // do the delete int c = _viewManager.ViewDelete(filter); if (c > 0) { if (Global.SkipDocsOnViewInsert) { // save this filter to docs View_delete vd = new View_delete(); LINQString lq = new LINQString(); lq.Visit(filter); vd.Filter = lq.sb.ToString(); vd.Viewname = _viewManager.GetViewName(typeof(TRowSchema)); _objStore.SetObject(vd.ID, vd); } } return c; } /// /// Direct delete from a view /// /// /// /// public int ViewDelete(string viewname, string filter) { // do the delete int c = _viewManager.ViewDelete(viewname, filter); if (c > 0) { if (Global.SkipDocsOnViewInsert) { // save this filter to docs View_delete vd = new View_delete(); vd.Filter = filter; vd.Viewname = viewname; _objStore.SetObject(vd.ID, vd); } } return c; } /// /// Direct insert into a view /// /// /// /// /// public bool ViewInsert(Guid id, TRowSchema row) { string vn = _viewManager.GetViewName(typeof(TRowSchema)); if (vn != "") { if (_viewManager.ViewInsert(id, row)) { if (Global.SkipDocsOnViewInsert == false) { View_insert vi = new View_insert(); vi.ID = id; vi.Viewname = vn; vi.RowObject = row; _objStore.SetObject(vi.ID, vi); } return true; } } return false; } /// /// Direct insert into a view /// /// /// /// /// public bool ViewInsert(string viewname, Guid id, object row) { if (_viewManager.ViewInsert(viewname, id, row)) { if (Global.SkipDocsOnViewInsert == false) { View_insert vi = new View_insert(); vi.ID = id; vi.Viewname = viewname; vi.RowObject = row; _objStore.SetObject(vi.ID, vi); } return true; } return false; } /// /// Total number of documents in the storage file including duplicates /// /// public long DocumentCount() { return _objStore.Count(); } public IKeyStoreHF GetKVHF() { return _objHF; } public object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args) { return func(this, filter, args).ToArray(); } public object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args) { LINQString ls = new LINQString(); ls.Visit(filter); return func(this, ls.sb.ToString(), args).ToArray(); } #endregion #region [ P R I V A T E M E T H O D S ] internal string GetViewName(Type type) { return _viewManager.GetViewName(type); } private bool SaveToView(Guid docid, T data, List list) { if (list != null) foreach (string name in list) { bool ret = _viewManager.InsertTransaction(name, docid, data); if (ret == false) return false; } return true; } private bool SaveInOtherViewsTransaction(Guid docid, T data) { List list = _viewManager.GetOtherViewsList(data.GetType()); return SaveToView(docid, data, list); } private bool SaveToConsistentViewsTransaction(Guid docid, T data) { List list = _viewManager.GetConsistentViews(data.GetType()); return SaveToView(docid, data, list); } private bool SaveInPrimaryViewTransaction(string viewname, Guid docid, T data) { return _viewManager.InsertTransaction(viewname, docid, data); } private static void PumpDataForBackup(Stream input, Stream output) { byte[] bytes = new byte[4096 * 2]; int n; while ((n = input.Read(bytes, 0, bytes.Length)) != 0) output.Write(bytes, 0, n); } private static void CompressForBackup(Stream source, Stream destination) { using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress)) PumpDataForBackup(source, gz); } private static void DecompressForRestore(Stream source, Stream destination) { using (GZipStream gz = new GZipStream(source, CompressionMode.Decompress)) PumpDataForBackup(gz, destination); } private void SaveToConsistentViews(Guid docid, T data) { List list = _viewManager.GetConsistentViews(data.GetType()); if (list != null) foreach (string name in list) { //_log.Info("Saving to consistent view : " + name); _viewManager.Insert(name, docid, data); } } private object CreateObject(byte[] b) { if (b[0] < 32) return fastBinaryJSON.BJSON.ToObject(b); else return fastJSON.JSON.ToObject(Encoding.ASCII.GetString(b)); } private void SaveInOtherViews(Guid docid, T data) { List list = _viewManager.GetOtherViewsList(data.GetType()); if (list != null) foreach (string name in list) _viewManager.Insert(name, docid, data); } private void SaveInPrimaryView(string viewname, Guid docid, T data) { _viewManager.Insert(viewname, docid, data); } private void Initialize() { // read raptordb.config here (running parameters) if (File.Exists(_Path + "RaptorDB.config")) fastJSON.JSON.FillObject(new Global(), File.ReadAllText(_Path + "RaptorDB.config")); Directory.CreateDirectory(_Path + "Data"); Directory.CreateDirectory(_Path + "Data" + _S + "Fulltext"); Directory.CreateDirectory(_Path + "Views"); Directory.CreateDirectory(_Path + "Logs"); Directory.CreateDirectory(_Path + "Temp"); Directory.CreateDirectory(_Path + "Backup"); Directory.CreateDirectory(_Path + "Restore"); Directory.CreateDirectory(_Path + "Restore" + _S + "Done"); // load logger LogManager.Configure(_Path + "Logs" + _S + "log.txt", 500, false); _log.Debug("\r\n\r\nRaptorDB starting..."); _log.Debug("RaptorDB data folder = " + _Path); // check doc & file storage file version and upgrade if needed here int v = StorageFile.GetStorageFileHeaderVersion(_Path + "Data" + _S + "data"); if (v < StorageFile._CurrentVersion) UpgradeStorageFile(_Path + "Data" + _S + "data", v); v = StorageFile.GetStorageFileHeaderVersion(_Path + "Data" + _S + "files"); if (v < StorageFile._CurrentVersion) UpgradeStorageFile(_Path + "Data" + _S + "files", v); // upgrade old mgidx files if (File.Exists(_Path + "Data" + _S + "RaptorDB.version") == false) { RebuildDataFiles(); } File.WriteAllText(_Path + "data" + _S + "RaptorDB.version", "" + _RaptorDBVersion); _objStore = new KeyStore(_Path + "Data" + _S + "data", true); _fileStore = new KeyStore(_Path + "Data" + _S + "files", true); _objHF = new KeyStoreHF(_Path + "DataHF"); _viewManager = new Views.ViewManager(_Path + "Views", _objStore, _objHF, _tokenizer); // load _LastFulltextIndexed if (File.Exists(_Path + "Data" + _S + "Fulltext" + _S + "_fulltext.rec")) { byte[] b = File.ReadAllBytes(_Path + "Data" + _S + "Fulltext" + _S + "_fulltext.rec"); _LastFulltextIndexed = Helper.ToInt32(b, 0, false); } // load _LastRecordNumberProcessed if (File.Exists(_Path + "Data" + _S + "_lastrecord.rec")) { byte[] b = File.ReadAllBytes(_Path + "Data" + _S + "_lastrecord.rec"); _LastRecordNumberProcessed = Helper.ToInt32(b, 0, false); } // load _LastBackupRecordNumber if (File.Exists(_Path + "Backup" + _S + "LastBackupRecord.rec")) { byte[] b = File.ReadAllBytes(_Path + "Backup" + _S + "LastBackupRecord.rec"); _LastBackupRecordNumber = Helper.ToInt32(b, 0, false); } _CurrentRecordNumber = _objStore.RecordCount(); otherviews = this.GetType().GetMethod("SaveInOtherViews", BindingFlags.Instance | BindingFlags.NonPublic); save = this.GetType().GetMethod("Save", BindingFlags.Instance | BindingFlags.Public); saverep = this.GetType().GetMethod("SaveReplicationObject", BindingFlags.Instance | BindingFlags.NonPublic); _fulltextindex = new FullTextIndex(_Path + "Data" + _S + "Fulltext", "fulltext", true, false, _tokenizer); // start backround save to views _saveTimer = new System.Timers.Timer(Global.BackgroundSaveViewTimer * 1000); _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed); _saveTimer.Enabled = true; _saveTimer.AutoReset = true; _saveTimer.Start(); // start full text timer _fulltextTimer = new System.Timers.Timer(Global.FullTextTimerSeconds * 1000); _fulltextTimer.Elapsed += new System.Timers.ElapsedEventHandler(_fulltextTimer_Elapsed); _fulltextTimer.Enabled = true; _fulltextTimer.AutoReset = true; _fulltextTimer.Start(); // start free memory timer _freeMemTimer = new System.Timers.Timer(Global.FreeMemoryTimerSeconds * 1000); _freeMemTimer.Elapsed += new System.Timers.ElapsedEventHandler(_freeMemTimer_Elapsed); _freeMemTimer.Enabled = true; _freeMemTimer.AutoReset = true; _freeMemTimer.Start(); // start inbox procesor timer _processinboxTimer = new System.Timers.Timer(Global.ProcessInboxTimerSeconds * 1000); _processinboxTimer.Elapsed += new System.Timers.ElapsedEventHandler(_processinboxTimer_Elapsed); _processinboxTimer.Enabled = true; _processinboxTimer.AutoReset = true; _processinboxTimer.Start(); // start cron daemon _cron = new CronDaemon(); _cron.AddJob(Global.BackupCronSchedule, () => this.Backup()); // compile & register view files CompileAndRegisterScriptViews(_Path + "Views"); if (File.Exists(_Path + "RaptorDB-Replication.config")) { // if replication.config exists -> start replication server _repserver = new Replication.ReplicationServer(_Path, File.ReadAllText(_Path + "RaptorDB-Replication.config"), _objStore); } else if (File.Exists(_Path + "RaptorDB-Branch.config")) { // if branch.config exists -> start replication client _repclient = new Replication.ReplicationClient(_Path, File.ReadAllText(_Path + "RaptorDB-Branch.config"), _objStore); } if (Global.EnableWebStudio) { _log.Debug("Enabling WEBSTUDIO on port : " + Global.WebStudioPort); _restServer = new rdbRest(Global.WebStudioPort, this, _Path, Global.LocalOnlyWebStudio); } } private void RebuildDataFiles() { try { var data = _Path + "data" + _S; var tmp = data + "temp" + _S; _log.Debug("Rebuilding MGDAT files..."); _log.Debug(" deleting Fulltext folder."); // delete "fulltext" folder -> will rebuild automatically Directory.Delete(data + "Fulltext", true); _log.Debug(" moving mgdat files to temp folder."); Directory.CreateDirectory(tmp); File.Move(data + "data.mgdat", tmp + "data.mgdat"); File.Move(data + "files.mgdat", tmp + "files.mgdat"); // delete all index files -> keep data.mgdat & files.mgdat Directory.GetFiles(data, "*.*").ToList().ForEach(x => File.Delete(x)); // upgrade by rebuilding indexes var sf = StorageFile.ReadForward(tmp + "data.mgdat"); var obstore = new KeyStore(data + "data", true); _log.Debug(" rebuilding index for data.mgdat"); _log.Debug(" docs count = " + sf.Count()); foreach (var i in sf.ReadOnlyEnumerate()) { if (i.meta.isDeleted) obstore.Delete(i.meta.key); else { var o = CreateObject(i.data); obstore.SetObject(i.meta.key, o); } } obstore.Shutdown(); sf.Shutdown(); sf = StorageFile.ReadForward(tmp + "files.mgdat"); obstore = new KeyStore(data + "files", true); _log.Debug(" rebuilding index for files.mgdat"); _log.Debug(" files count = " + sf.Count()); foreach (var i in sf.ReadOnlyEnumerate()) { if (i.meta.isDeleted) obstore.Delete(i.meta.key); else { var o = CreateObject(i.data); obstore.SetObject(i.meta.key, o); } } obstore.Shutdown(); sf.Shutdown(); File.WriteAllText(data + "RaptorDB.version", "" + _RaptorDBVersion); Directory.Delete(tmp, true); } catch (Exception ex) { _log.Error(ex); } } object _inboxlock = new object(); void _processinboxTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { lock (_inboxlock) { string d = _Path + "Replication" + _S + "Inbox"; if (Directory.Exists(d) == false) return; // start inbox processing timer ProcessReplicationInbox(d); foreach (var f in Directory.GetDirectories(d)) ProcessReplicationInbox(f); } } private void CompileAndRegisterScriptViews(string viewfolder) { // compile & register views string[] files = Directory.GetFiles(viewfolder, "*.view"); MethodInfo register = this.GetType().GetMethod("RegisterView", BindingFlags.Instance | BindingFlags.Public); foreach (var fn in files) { Assembly a = CompileScript(fn); if (a != null) { foreach (var t in a.GetTypes()) { foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false)) { try { object o = Activator.CreateInstance(t); // handle types when view also Type[] args = t.GetGenericArguments(); if (args.Length == 0) args = t.BaseType.GetGenericArguments(); Type tt = args[0]; var m = register.MakeGenericMethod(new Type[] { tt }); m.Invoke(this, new object[] { o }); } catch (Exception ex) { _log.Error(ex); } } } } } } private Assembly CompileScript(string file) { try { _log.Debug("Compiling script view : " + file); CodeDomProvider compiler = CodeDomProvider.CreateProvider("CSharp"); CompilerParameters compilerparams = new CompilerParameters(); compilerparams.GenerateInMemory = false; compilerparams.GenerateExecutable = false; compilerparams.OutputAssembly = file.Replace(".view", ".dll"); compilerparams.CompilerOptions = "/optimize"; Regex regex = new Regex( @"\/\/\s*ref\s*\:\s*(?.*)", System.Text.RegularExpressions.RegexOptions.IgnoreCase); compilerparams.ReferencedAssemblies.Add(typeof(View<>).Assembly.Location); compilerparams.ReferencedAssemblies.Add(typeof(object).Assembly.Location); compilerparams.ReferencedAssemblies.Add(typeof(ICustomTypeDescriptor).Assembly.Location); foreach (Match m in regex.Matches(File.ReadAllText(file))) { string str = m.Groups["refs"].Value.Trim(); #pragma warning disable 618 Assembly a = Assembly.LoadWithPartialName(Path.GetFileNameWithoutExtension(str));//load from GAC if possible #pragma warning restore 618 if (a != null) compilerparams.ReferencedAssemblies.Add(a.Location); else { string assm = Path.GetDirectoryName(this.GetType().Assembly.Location) + _S + str; a = Assembly.LoadFrom(assm); if (a != null) compilerparams.ReferencedAssemblies.Add(a.Location); else _log.Error("unable to find referenced file for view compiling : " + str); } } CompilerResults results = compiler.CompileAssemblyFromFile(compilerparams, file); if (results.Errors.HasErrors == true) { _log.Error("Error compiling view definition : " + file); foreach (var e in results.Errors) _log.Error(e.ToString()); return null; } return results.CompiledAssembly; } catch (Exception ex) { _log.Error("Error compiling view definition : " + file); _log.Error(ex); return null; } } void _freeMemTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { FreeMemory(); } private void UpgradeStorageFile(string filename, int ver) { _log.Debug("Upgrading storage file version from " + ver + " to " + StorageFile._CurrentVersion + " on file : " + filename); throw new Exception("not implemented yet - contact the author if you need this functionality"); // read from one file and write to the other } private object _slock = new object(); private void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { if (_shuttingdown) return; if (Global.BackgroundSaveToOtherViews == false) return; if (_CurrentRecordNumber == 0) return; if (_CurrentRecordNumber == _LastRecordNumberProcessed) return; lock (_slock) { int batch = Global.BackgroundViewSaveBatchSize; while (batch > 0) { if (_shuttingdown) return; while (_pauseindexer) Thread.Sleep(0); if (_CurrentRecordNumber == _LastRecordNumberProcessed) return; _LastRecordNumberProcessed++; StorageItem meta = null; object obj = _objStore.GetObject(_LastRecordNumberProcessed, out meta); if (meta != null && meta.isDeleted) _viewManager.Delete(meta.key); else { if (obj == null) { _log.Debug("byte[] is null"); _log.Debug("curr rec = " + _CurrentRecordNumber); _log.Debug("last rec = " + _LastRecordNumberProcessed); continue; } var m = otherviews.MakeGenericMethod(new Type[] { obj.GetType() }); m.Invoke(this, new object[] { meta.key, obj }); } batch--; } } } private object _flock = new object(); private Regex _jsonfilter = new Regex("[\\[\\]\"{}:,]", RegexOptions.Compiled); private rdbRest _restServer = null; private void _fulltextTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { if (_shuttingdown) return; if (_CurrentRecordNumber == 0) return; if (_CurrentRecordNumber == _LastFulltextIndexed) return; lock (_flock) { int batch = Global.BackgroundFullTextIndexBatchSize; while (batch > 0) { if (_shuttingdown) return; //_log.Debug("batch full text indexing..."); while (_pauseindexer) Thread.Sleep(0); if (_CurrentRecordNumber == _LastFulltextIndexed) return; _LastFulltextIndexed++; StorageItem meta = null; object obj = _objStore.GetObject(_LastFulltextIndexed, out meta); if (meta != null && meta.isDeleted == false) { if (obj != null) { // normal string and normal guid string json = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseEscapedUnicode = false, UseFastGuid = false }); json = _jsonfilter.Replace(json, " "); // filter out json characters _fulltextindex.Set(json, _LastFulltextIndexed); } } batch--; } return; } } private MethodInfo GetSave(Type type) { MethodInfo m = null; if (_savecache.TryGetValue(type, out m)) return m; m = save.MakeGenericMethod(new Type[] { type }); _savecache.Add(type, m); return m; } private MethodInfo GetSaveReplicate(Type type) { MethodInfo m = null; if (_saverepcache.TryGetValue(type, out m)) return m; m = saverep.MakeGenericMethod(new Type[] { type }); _saverepcache.Add(type, m); return m; } private void CreateTemplateConfigFiles() { if (File.Exists(_Path + "RaptorDB.config") == false) File.WriteAllText(_Path + "-RaptorDB.config", fastJSON.JSON.ToNiceJSON(new Global(), new fastJSON.JSONParameters { UseExtensions = false })); if (File.Exists(_Path + "RaptorDB-Branch.config") == false) File.WriteAllText(_Path + "-RaptorDB-Branch.config", fastJSON.JSON.ToNiceJSON(new Replication.ClientConfiguration(), new fastJSON.JSONParameters { UseExtensions = false })); if (File.Exists(_Path + "RaptorDB-Replication.config") == false) { Replication.ServerConfiguration s = new Replication.ServerConfiguration(); s.What.Add(new Replication.WhatItem { Name = "default", PackageItemLimit = 10000, Version = 1, B2HQtypes = new List { "*" }, HQ2Btypes = new List { "*" } }); s.What.Add(new Replication.WhatItem { Name = "b2", PackageItemLimit = 10000, Version = 1, B2HQtypes = new List { "*" }, HQ2Btypes = new List { "config.*" } }); s.Where.Add(new Replication.WhereItem { BranchName = "b1", Password = "123", When = "*/5 * * * *", What = "default" }); s.Where.Add(new Replication.WhereItem { BranchName = "b2", Password = "321", When = "*/20 * * * *", What = "b2" }); File.WriteAllText(_Path + "-RaptorDB-Replication.config", fastJSON.JSON.ToNiceJSON(s, new fastJSON.JSONParameters { UseExtensions = false })); } } internal long GetDataFolderSize() { // get data folder size return GetDirectorySize(_Path); } internal long GetDirectorySize(string path) { long b = 0; foreach (var p in Directory.GetDirectories(path)) { b += GetDirectorySize(p); string[] a = Directory.GetFiles(p, "*.*"); foreach (string name in a) { FileInfo info = new FileInfo(name); b += info.Length; } } return b; } internal object GetAssemblyForView(string viewname, out string typename) { return _viewManager.GetAssemblyForView(viewname, out typename); } public T Fetch(Guid docID) where T : class { return Fetch(docID) as T; } public void FreeMemory() { long l = GC.GetTotalMemory(true) / (1024 * 1024); _log.Debug("GC.GetTotalMemory() = " + l.ToString("#,0")); if (l > Global.MemoryLimit) { _log.Debug("Freeing memory on " + Global.MemoryLimit.ToString("#,0") + " limit ..."); _viewManager.FreeMemory(); _fulltextindex.FreeMemory(); _objStore.FreeMemory(); _fileStore.FreeMemory(); _objHF.FreeMemory(); GC.Collect();// GC.MaxGeneration); } } public void SaveToDocsOnViewInsert(bool yes) { Global.SkipDocsOnViewInsert = yes; } #endregion } } ================================================ FILE: RaptorDB/RaptorDB.csproj ================================================  {45F6BE30-989A-4749-B6A0-69099C8661F4} Debug AnyCPU RaptorDB False Library RaptorDB OnBuildSuccess v4.0 false true 0 false true true bin\Debug\ true 285212672 false true 4096 false false false false false 4 Full prompt false TRACE;DEBUG bin\Release\ false 285212672 false false 4096 false true false false false 4 none prompt false Auto AnyCPU TRACE Project ..\raptordb.snk BuildVersion.cs Code True True Resources.resx Code RaptorDB_Doc.nuspec Designer {32331D51-5BE0-41E2-AF1A-9B086C5AE809} RaptorDB.Common ResXFileCodeGenerator Resources.Designer.cs Designer md "$(SolutionDir)output\net40" copy "$(TargetPath)" "$(SolutionDir)output\net40\$(TargetFileName)" ================================================ FILE: RaptorDB/RaptorDBServer.cs ================================================ using System; using System.Collections.Generic; using System.Text; using RaptorDB.Common; using System.Reflection; using System.IO; using System.Threading.Tasks; namespace RaptorDB { public delegate void Handler(Packet data, ReturnPacket ret); public class instance_handler { public bool Initialized = false; public string dbpath; public MethodInfo register; public MethodInfo save; public RaptorDB rdb; public DateTime lastUsed = DateTime.MinValue; public bool hasExtensions = false; public SafeDictionary saveCache = new SafeDictionary(); public SafeDictionary ssideCache = new SafeDictionary(); public SafeDictionary sswcideCache = new SafeDictionary(); } public class RaptorDBServer { public RaptorDBServer(int port, string DataPath) { _path = Directory.GetCurrentDirectory(); AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve); _server = new NetworkServer(); if (_S == "/")// unix system _datapath = DataPath.Replace("\\", "/"); else _datapath = DataPath; if (_datapath.EndsWith(_S) == false) _datapath += _S; // check if "instances" folder exist -> multi instance mode if (Directory.Exists(_datapath + "instances")) { _log.Debug("Insances exist, loading..."); _multiInstance = true; foreach (var d in Directory.GetDirectories(_datapath + "instances")) { var dn = new DirectoryInfo(d); var i = new instance_handler(); i.dbpath = d; if (Directory.Exists(d + _S + "Extensions")) i.hasExtensions = true; _instances.Add(dn.Name.ToLower(), i); } } _defaultInstance.rdb = RaptorDB.Open(DataPath); _defaultInstance.register = _defaultInstance.rdb.GetType().GetMethod("RegisterView", BindingFlags.Instance | BindingFlags.Public); _defaultInstance.save = _defaultInstance.rdb.GetType().GetMethod("Save", BindingFlags.Instance | BindingFlags.Public); Initialize(); _server.Start(port, processpayload); // add timer to cleanup connected clients _concleanuptimer = new System.Timers.Timer(30 * 1000); _concleanuptimer.AutoReset = true; _concleanuptimer.Enabled = true; _concleanuptimer.Elapsed += _concleanuptimer_Elapsed; _unusedinstancetimer = new System.Timers.Timer(300 * 1000);// FIX : configuration here _unusedinstancetimer.AutoReset = true; _unusedinstancetimer.Enabled = true; _unusedinstancetimer.Elapsed += _unusedinstancetimer_Elapsed; } private object _lock = new object(); private void _unusedinstancetimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { lock (_lock) { bool freed = false; // clear unused rdb instances if (_multiInstance) { foreach (var i in _instances) { if (i.Value.rdb != null && FastDateTime.Now.Subtract(i.Value.lastUsed).TotalMinutes > 60) // FIX : configuration here { var r = i.Value; r.rdb.Shutdown(); r.Initialized = false; r.register = null; r.save = null; r.saveCache = new SafeDictionary(); r.ssideCache = new SafeDictionary(); r.sswcideCache = new SafeDictionary(); r.rdb = null; freed = true; } } if (freed) GC.Collect(); } } } void _concleanuptimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { _connectedClients.Clear(); } private bool _multiInstance = false; private SafeDictionary _instances = new SafeDictionary(); private instance_handler _defaultInstance = new instance_handler(); private string _S = Path.DirectorySeparatorChar.ToString(); private Dictionary _users = new Dictionary(); private string _path = ""; private string _datapath = ""; private ILog _log = LogManager.GetLogger(typeof(RaptorDBServer)); private NetworkServer _server; //private RaptorDB _raptor; //private MethodInfo register = null; //private MethodInfo save = null; //private SafeDictionary _savecache = new SafeDictionary(); //private SafeDictionary _ssidecache = new SafeDictionary(); //private SafeDictionary _sswcidecache = new SafeDictionary(); private Dictionary _handlers = new Dictionary(); private const string _RaptorDB_users_config = "RaptorDB-Users.config"; private SafeDictionary _connectedClients = new SafeDictionary(); private System.Timers.Timer _concleanuptimer; private System.Timers.Timer _unusedinstancetimer; public int ConnectedClients { get { return _connectedClients.Count(); } } private Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args) // FIX : handle instance?? { if (File.Exists(args.Name)) return Assembly.LoadFrom(args.Name); string[] ss = args.Name.Split(','); string fname = ss[0] + ".dll"; if (File.Exists(fname)) return Assembly.LoadFrom(fname); fname = "Extensions" + _S + fname; if (File.Exists(fname)) return Assembly.LoadFrom(fname); else return null; } private MethodInfo GetSave(Type type) { MethodInfo m = null; if (_defaultInstance.saveCache.TryGetValue(type, out m)) return m; m = _defaultInstance.save.MakeGenericMethod(new Type[] { type }); _defaultInstance.saveCache.Add(type, m); return m; } public void Shutdown() { WriteUsers(); _server.Stop(); _defaultInstance.rdb.Shutdown(); foreach (var i in _instances) { _log.Debug("Shutting down instance : " + i.Key); if (i.Value.rdb != null) i.Value.rdb.Shutdown(); } } private void WriteUsers() { // write users to user.config file StringBuilder sb = new StringBuilder(); sb.AppendLine("# FORMAT : username , pasword hash"); sb.AppendLine("# To disable a user comment the line with the '#'"); foreach (var kv in _users) { sb.AppendLine(kv.Key + " , " + kv.Value); } File.WriteAllText(_datapath + _RaptorDB_users_config, sb.ToString()); } private object processpayload(object data) { Packet p = (Packet)data; ReturnPacket ret = new ReturnPacket(true); if (Authenticate(p) == false) return new ReturnPacket(false, "Authentication failed"); if (p.Command == "_close") { _connectedClients.Remove(p.ClientID); return ret; } else _connectedClients.Add(p.ClientID, true); try { Handler d = null; if (_handlers.TryGetValue(p.Command, out d)) d(p, ret); else _log.Error("Command handler not found : " + p.Command); } catch (Exception ex) { ret.OK = false; _log.Error(ex); } return ret; } private RaptorDB GetInstance(string name) { // load or get instance instance_handler inst = null; _instances.TryGetValue(name.ToLower(), out inst); if(inst==null) { // no instance found -> err _log.Debug("instance name not found : " + name); return null; } if(inst.rdb==null) { // try loading raptordb instance var r = RaptorDB.Open(inst.dbpath); inst.rdb = r; // fix : create register and save //inst.register //inst.save if (inst.hasExtensions) { // fix: load extension folder } else { // fix: load default extenstions } } return inst.rdb; } private void InitializeCommandsDictionary() { // FIX : route to instance on p.InstanceName _handlers.Add("" + COMMANDS.Save, (p, ret) => { var m = GetSave(p.Data.GetType()); ret.OK = true; m.Invoke(_defaultInstance.rdb, new object[] { p.Docid, p.Data }); }); _handlers.Add("" + COMMANDS.SaveBytes, (p, ret) => { ret.OK = _defaultInstance.rdb.SaveBytes(p.Docid, (byte[])p.Data); }); _handlers.Add("" + COMMANDS.QueryType, (p, ret) => { var param = (object[])p.Data; Type t = Type.GetType((string)param[0]); string viewname = _defaultInstance.rdb.GetViewName(t); ret.OK = true; ret.Data = _defaultInstance.rdb.Query(viewname, (string)param[1], p.Start, p.Count, p.OrderBy); }); _handlers.Add("" + COMMANDS.QueryStr, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.Query(p.Viewname, (string)p.Data, p.Start, p.Count, p.OrderBy); }); _handlers.Add("" + COMMANDS.Fetch, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.Fetch(p.Docid); }); _handlers.Add("" + COMMANDS.FetchBytes, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchBytes(p.Docid); }); _handlers.Add("" + COMMANDS.Backup, (p, ret) => { ret.OK = _defaultInstance.rdb.Backup(); }); _handlers.Add("" + COMMANDS.Delete, (p, ret) => { ret.OK = _defaultInstance.rdb.Delete(p.Docid); }); _handlers.Add("" + COMMANDS.DeleteBytes, (p, ret) => { ret.OK = _defaultInstance.rdb.DeleteBytes(p.Docid); }); _handlers.Add("" + COMMANDS.Restore, (p, ret) => { ret.OK = true; Task.Factory.StartNew(() => _defaultInstance.rdb.Restore()); }); _handlers.Add("" + COMMANDS.AddUser, (p, ret) => { var param = (object[])p.Data; ret.OK = AddUser((string)param[0], (string)param[1], (string)param[2]); }); _handlers.Add("" + COMMANDS.ServerSide, (p, ret) => { var param = (object[])p.Data; ret.OK = true; ret.Data = _defaultInstance.rdb.ServerSide(GetServerSideFuncCache(param[0].ToString(), param[1].ToString()), param[2].ToString()); }); _handlers.Add("" + COMMANDS.FullText, (p, ret) => { var param = (object[])p.Data; ret.OK = true; ret.Data = _defaultInstance.rdb.FullTextSearch("" + param[0]); }); _handlers.Add("" + COMMANDS.CountType, (p, ret) => { // count type var param = (object[])p.Data; Type t = Type.GetType((string)param[0]); string viewname2 = _defaultInstance.rdb.GetViewName(t); ret.OK = true; ret.Data = _defaultInstance.rdb.Count(viewname2, (string)param[1]); }); _handlers.Add("" + COMMANDS.CountStr, (p, ret) => { // count str ret.OK = true; ret.Data = _defaultInstance.rdb.Count(p.Viewname, (string)p.Data); }); _handlers.Add("" + COMMANDS.GCount, (p, ret) => { Type t = Type.GetType(p.Viewname); string viewname3 = _defaultInstance.rdb.GetViewName(t); ret.OK = true; ret.Data = _defaultInstance.rdb.Count(viewname3, (string)p.Data); }); _handlers.Add("" + COMMANDS.DocHistory, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchHistory(p.Docid); }); _handlers.Add("" + COMMANDS.FileHistory, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchBytesHistory(p.Docid); }); _handlers.Add("" + COMMANDS.FetchVersion, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchVersion((int)p.Data); }); _handlers.Add("" + COMMANDS.FetchFileVersion, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchBytesVersion((int)p.Data); }); _handlers.Add("" + COMMANDS.CheckAssembly, (p, ret) => { ret.OK = true; string typ = ""; ret.Data = _defaultInstance.rdb.GetAssemblyForView(p.Viewname, out typ); ret.Error = typ; }); _handlers.Add("" + COMMANDS.FetchHistoryInfo, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchHistoryInfo(p.Docid); }); _handlers.Add("" + COMMANDS.FetchByteHistoryInfo, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.FetchBytesHistoryInfo(p.Docid); }); _handlers.Add("" + COMMANDS.ViewDelete, (p, ret) => { ret.OK = true; var param = (object[])p.Data; ret.Data = _defaultInstance.rdb.ViewDelete((string)param[0], (string)param[1]); }); _handlers.Add("" + COMMANDS.ViewDelete_t, (p, ret) => { ret.OK = true; var param = (object[])p.Data; Type t = Type.GetType((string)param[0]); string viewname4 = _defaultInstance.rdb.GetViewName(t); ret.Data = _defaultInstance.rdb.ViewDelete(viewname4, (string)param[1]); }); _handlers.Add("" + COMMANDS.ViewInsert, (p, ret) => { ret.OK = true; var param = (object[])p.Data; ret.Data = _defaultInstance.rdb.ViewInsert((string)param[0], p.Docid, param[1]); }); _handlers.Add("" + COMMANDS.ViewInsert_t, (p, ret) => { ret.OK = true; var param = (object[])p.Data; Type t = Type.GetType((string)param[0]); string viewname5 = _defaultInstance.rdb.GetViewName(t); ret.Data = _defaultInstance.rdb.ViewInsert(viewname5, p.Docid, param[1]); }); _handlers.Add("" + COMMANDS.DocCount, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.DocumentCount(); }); _handlers.Add("" + COMMANDS.GetObjectHF, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.GetKVHF().GetObjectHF((string)p.Data); }); _handlers.Add("" + COMMANDS.SetObjectHF, (p, ret) => { ret.OK = true; var param = (object[])p.Data; _defaultInstance.rdb.GetKVHF().SetObjectHF((string)param[0], param[1]); }); _handlers.Add("" + COMMANDS.DeleteKeyHF, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.GetKVHF().DeleteKeyHF((string)p.Data); }); _handlers.Add("" + COMMANDS.CountHF, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.GetKVHF().CountHF(); }); _handlers.Add("" + COMMANDS.ContainsHF, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.GetKVHF().ContainsHF((string)p.Data); }); _handlers.Add("" + COMMANDS.GetKeysHF, (p, ret) => { ret.OK = true; ret.Data = _defaultInstance.rdb.GetKVHF().GetKeysHF(); }); _handlers.Add("" + COMMANDS.CompactStorageHF, (p, ret) => { ret.OK = true; _defaultInstance.rdb.GetKVHF().CompactStorageHF(); }); _handlers.Add("" + COMMANDS.IncrementHF, (p, ret) => { ret.OK = true; var param = (object[])p.Data; if (param[1] is int) ret.Data = _defaultInstance.rdb.GetKVHF().Increment((string)param[0], (int)param[1]); else ret.Data = _defaultInstance.rdb.GetKVHF().Increment((string)param[0], (decimal)param[1]); }); _handlers.Add("" + COMMANDS.DecrementHF, (p, ret) => { ret.OK = true; var param = (object[])p.Data; if (param[1] is int) ret.Data = _defaultInstance.rdb.GetKVHF().Decrement((string)param[0], (int)param[1]); else ret.Data = _defaultInstance.rdb.GetKVHF().Decrement((string)param[0], (decimal)param[1]); }); _handlers.Add("" + COMMANDS.ServerSideWithArgs, (p, ret) => { var param = (object[])p.Data; ret.OK = true; ret.Data = _defaultInstance.rdb.ServerSide(GetServerSideFuncWithArgsCache(param[0].ToString(), param[1].ToString()), param[2].ToString(), param[3]); }); _handlers.Add("" + COMMANDS.FreeMemory, (p, ret) => { ret.OK = true; _log.Debug("Free memory called from client"); _defaultInstance.rdb.FreeMemory(); }); } private ServerSideFuncWithArgs GetServerSideFuncWithArgsCache(string type, string method) { ServerSideFuncWithArgs func; _log.Debug("Calling Server side Function with args : " + method + " on type " + type); if (_defaultInstance.sswcideCache.TryGetValue(type + method, out func) == false) { Type tt = Type.GetType(type); func = (ServerSideFuncWithArgs)Delegate.CreateDelegate(typeof(ServerSideFuncWithArgs), tt, method); _defaultInstance.sswcideCache.Add(type + method, func); } return func; } private ServerSideFunc GetServerSideFuncCache(string type, string method) { ServerSideFunc func; _log.Debug("Calling Server side Function : " + method + " on type " + type); if (_defaultInstance.ssideCache.TryGetValue(type + method, out func) == false) { Type tt = Type.GetType(type); func = (ServerSideFunc)Delegate.CreateDelegate(typeof(ServerSideFunc), tt, method); _defaultInstance.ssideCache.Add(type + method, func); } return func; } private uint GenHash(string user, string pwd) { return Helper.MurMur.Hash(Encoding.UTF8.GetBytes(user.ToLower() + "|" + pwd)); } private bool AddUser(string user, string oldpwd, string newpwd) { uint hash = 0; if (_users.TryGetValue(user.ToLower(), out hash) == false) { _users.Add(user.ToLower(), GenHash(user, newpwd)); return true; } if (hash == GenHash(user, oldpwd)) { _users[user.ToLower()] = GenHash(user, newpwd); return true; } return false; } private bool Authenticate(Packet p) { uint pwd; if (_users.TryGetValue(p.Username.ToLower(), out pwd)) { uint hash = uint.Parse(p.PasswordHash); if (hash == pwd) return true; } _log.Debug("Authentication failed for '" + p.Username + "' hash = " + p.PasswordHash); return false; } private void Initialize() { // load users here if (File.Exists(_datapath + _RaptorDB_users_config)) { foreach (string line in File.ReadAllLines(_datapath + _RaptorDB_users_config)) { if (line.Contains("#") == false) { string[] s = line.Split(','); _users.Add(s[0].Trim().ToLower(), uint.Parse(s[1].Trim())); } } } // add default admin user if not exists if (_users.ContainsKey("admin") == false) _users.Add("admin", GenHash("admin", "admin")); // exe folder // |-Extensions Directory.CreateDirectory(_path + _S + "Extensions"); // open extensions folder string path = _path + _S + "Extensions"; foreach (var f in Directory.GetFiles(path, "*.dll")) { // - load all dll files // - register views _log.Debug("loading dll for views : " + f); Assembly a = Assembly.Load(f); foreach (var t in a.GetTypes()) { foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false)) { try { object o = Activator.CreateInstance(t); // handle types when view also Type[] args = t.GetGenericArguments(); if (args.Length == 0) args = t.BaseType.GetGenericArguments(); Type tt = args[0]; var m = _defaultInstance.register.MakeGenericMethod(new Type[] { tt }); m.Invoke(_defaultInstance.rdb, new object[] { o }); } catch (Exception ex) { _log.Error(ex); } } } } InitializeCommandsDictionary(); } } } ================================================ FILE: RaptorDB/Replication/Configuration.cs ================================================ using System.Collections.Generic; namespace RaptorDB.Replication { public class WhereItem { public string BranchName; public string Password; public string What; public string When; } public class WhatItem { public WhatItem() { HQ2Btypes = new List(); B2HQtypes = new List(); } public string Name; public int Version = 1; public bool PropogateHQDeletes = true; public int PackageItemLimit = 10000; public List HQ2Btypes; public List B2HQtypes; } public class ServerConfiguration { public ServerConfiguration() { Where = new List(); What = new List(); ReplicationPort = 9999; } public int ReplicationPort; //public string EmbeddedClientHandler; public List Where; public List What; } public enum REPMODE { Branch, Server } public class ClientConfiguration { public ClientConfiguration() { ServerReplicationPort = 9999; } public string ServerAddress = ""; public int ServerReplicationPort; public string Password = ""; public string BranchName = ""; } } ================================================ FILE: RaptorDB/Replication/Packets.cs ================================================ namespace RaptorDB.Replication { public class ReplicationPacket { //public int number; public string passwordhash; public string branchname;// source name public uint datahash; public string filename; public object data; public string command; public int lastrecord; } } ================================================ FILE: RaptorDB/Replication/Readme.txt ================================================ DATA Folder | |- Replication > (branch mode) | |-: branch.dat | | . |- Inbox > | |-: 0000000n.mgdat.gz, | |- Outbox > - if inbox contains : "0000000n.counter" then error occurred and the text is in "0000000n.error.txt" - you can skip the offending document if you increment the "counter" file (when you can't overcome the exception) - files will be downloaded to the inbox folder in branch mode - "branch.dat" in the "Replication" folder stores counter information for replication DATA Folder | |- Replication > (HQ mode) | |-: BranchName1.last | | | |- Inbox > | | | . | |- BranchName1 > | | |-: 0000000n.mgdat.gz | | | | |- Outbox > | | | |- BranchName1 > | |- BranchName2 > - if inbox contains : "0000000n.counter" then error occurred and the text is in "0000000n.error.txt" - you can skip the offending document if you increment the "counter" file (when you can't overcome the exception) - files will be downloaded to the inbox folder in branch mode ================================================ FILE: RaptorDB/Replication/ReplicationClient.cs ================================================ using System; using System.IO; using System.IO.Compression; using RaptorDB.Common; using System.Text.RegularExpressions; namespace RaptorDB.Replication { public class ClientRepConfig { public bool isConfigured; public string whencron = "* * * * *"; public WhatItem what; public int lastHQCounter; public int lastCounter; public int outPackageNumber; public int inPackageNumber; public int lastPackageIndex; } public class ClientWhatWhenConfig { public WhatItem what; public string whencron; } /// /// Replication package processing is done in RaptorDB.cs /// internal class ReplicationClient { public ReplicationClient(string dataFolder, string config, IDocStorage docs) { _log.Debug("starting replication client : " + dataFolder); _docs = docs; _path = dataFolder; // read client config file _config = fastJSON.JSON.ToObject(config); Initialize(); } private void Initialize() { Directory.CreateDirectory(_path + "Replication"); Directory.CreateDirectory(_path + "Replication" + _S + "Inbox"); Directory.CreateDirectory(_path + "Replication" + _S + "Outbox"); _InboxPath = _path + "Replication" + _S + "Inbox" + _S; _OutboxPath = _path + "Replication" + _S + "Outbox" + _S; // setup cron job _cron = new CronDaemon(); _clientConfig = new ClientRepConfig(); // read what config if (File.Exists(_path + "Replication" + _S + "branch.dat")) _clientConfig = fastBinaryJSON.BJSON.ToObject(File.ReadAllBytes(_path + "Replication" + _S + "branch.dat")); // starting jobs _cron.AddJob(_clientConfig.whencron, Replicate); } private ILog _log = LogManager.GetLogger(typeof(ReplicationClient)); IDocStorage _docs; private CronDaemon _cron; private string _S = Path.DirectorySeparatorChar.ToString(); private NetworkClient _client; private Replication.ClientConfiguration _config; private ClientRepConfig _clientConfig; private string _path; private string _OutboxPath; private string _InboxPath; private int INTERNALLIMIT = Global.PackageSizeItemCountLimit; public void Shutdown() { if (_cron != null) _cron.Stop(); SaveConfig(); } private void SaveConfig() { if (_clientConfig == null) return; if (_clientConfig.isConfigured == false) return; // write config to disk byte[] b = fastBinaryJSON.BJSON.ToBJSON(_clientConfig); File.WriteAllBytes(_path + "Replication" + _S + "branch.dat", b); } private object _lock = new object(); private void Replicate() { lock (_lock) { try { if (ConnectToHQ()) { SendPackageToHQ(); GetPackageFormHQ(); } } catch (Exception ex) { _log.Error(ex); } finally { if (_client != null) { _client.Close(); _client = null; } } } } private void GetPackageFormHQ() { ReplicationPacket p = createpacket(); p.command = "getpackageforbranch"; p.lastrecord = _clientConfig.lastHQCounter; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) { if (ret.Data != null) { ReplicationPacket pack = (ReplicationPacket)ret.Data; if (pack.datahash == Helper.MurMur.Hash((byte[])pack.data)) { _log.Debug("package recieved from server : " + pack.filename); _log.Debug("package size : " + (pack.data as byte[]).Length.ToString("#,0")); File.WriteAllBytes(_InboxPath + pack.filename, (byte[])pack.data); p = createpacket(); p.command = "hqpackageok"; p.filename = pack.filename; p.lastrecord = pack.lastrecord; _clientConfig.lastHQCounter = pack.lastrecord; SaveConfig(); ret = (ReturnPacket)_client.Send(p); if (ret.OK) return; } } } } private void SendPackageToHQ() { string fn = CreatePackageForSend(); if (fn != "") { ReplicationPacket p = createpacket(); p.command = "packageforhq"; p.data = File.ReadAllBytes(fn); p.datahash = Helper.MurMur.Hash((byte[])p.data); ReturnPacket ret = (ReturnPacket)_client.Send(p); string path = Path.GetDirectoryName(fn); string fnn = Path.GetFileNameWithoutExtension(fn); foreach (var f in Directory.GetFiles(path, fnn + ".*")) File.Delete(f); } } private ReplicationPacket createpacket() { ReplicationPacket p = new ReplicationPacket(); p.branchname = _config.BranchName; p.passwordhash = Helper.MurMur.Hash(Helper.GetBytes(_config.BranchName + "|" + _config.Password)).ToString(); return p; } private bool ConnectToHQ() { if (_client == null) { _client = new NetworkClient(_config.ServerAddress, _config.ServerReplicationPort); } // authenticate and get branch config ReplicationPacket p = createpacket(); p.command = "getbranchconfig"; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) { ClientWhatWhenConfig c = (ClientWhatWhenConfig)ret.Data; _clientConfig.what = c.what; _clientConfig.isConfigured = true; if (_clientConfig.whencron != c.whencron) { _cron.Stop(); _clientConfig.whencron = c.whencron; _cron = new CronDaemon(); _cron.AddJob(_clientConfig.whencron, Replicate); } SaveConfig(); } return ret.OK; } private string CreatePackageForSend() { int maxc = INTERNALLIMIT; if (_clientConfig.what.PackageItemLimit > 0) maxc = _clientConfig.what.PackageItemLimit; string outFolder = _OutboxPath; int packageNumber = _clientConfig.outPackageNumber; int i = _clientConfig.lastCounter; string filename = outFolder + packageNumber.ToString("0000000000") + ".mgdat"; int total = _docs.RecordCount(); if (i < total) { StorageFile package = new StorageFile(filename, SF_FORMAT.JSON, true); while (maxc > 0 && i < total) { var meta = _docs.GetMeta(i); if (meta == null) break; if (meta.isReplicated == false && MatchType(meta.typename)) { object obj = _docs.GetObject(i, out meta); package.WriteObject(meta.key, obj); maxc--; } i++; } package.Shutdown(); packageNumber++; // compress the file using (FileStream read = File.OpenRead(filename)) using (FileStream outp = File.Create(filename + ".gz")) CompressForBackup(read, outp); // delete uncompressed file File.Delete(filename); _clientConfig.lastCounter = i; _clientConfig.outPackageNumber = packageNumber; SaveConfig(); return filename + ".gz"; } return ""; } private bool MatchType(string typename) { // match type filter foreach (var i in _clientConfig.what.B2HQtypes) { // do wildcard search Regex reg = new Regex("^" + i.Replace("*", ".*").Replace("?", "."), RegexOptions.IgnoreCase); if (reg.IsMatch(typename)) return true; } return false; } private static void CompressForBackup(Stream source, Stream destination) { using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress)) PumpDataForBackup(source, gz); } private static void PumpDataForBackup(Stream input, Stream output) { byte[] bytes = new byte[4096 * 2]; int n; while ((n = input.Read(bytes, 0, bytes.Length)) != 0) output.Write(bytes, 0, n); } } } ================================================ FILE: RaptorDB/Replication/ReplicationServer.cs ================================================ using System; using RaptorDB.Common; using System.IO; using System.IO.Compression; using System.Text.RegularExpressions; namespace RaptorDB.Replication { internal class ReplicationServer { public ReplicationServer(string datapath, string config, IDocStorage docs) { _docs = docs; _Path = datapath; Initialize(config); } IDocStorage _docs; private string _S = Path.DirectorySeparatorChar.ToString(); private string _Path; private ILog _log = LogManager.GetLogger(typeof(ReplicationServer)); private ServerConfiguration _config; private NetworkServer _server; private string _InboxPath; private string _OutboxPath; private int INTERNALLIMIT = Global.PackageSizeItemCountLimit; private SafeDictionary _branchLastDocs = new SafeDictionary(); private void Initialize(string config) { _log.Debug("Starting replication server..."); Directory.CreateDirectory(_Path + "Replication"); Directory.CreateDirectory(_Path + "Replication" + _S + "Inbox"); Directory.CreateDirectory(_Path + "Replication" + _S + "Outbox"); _InboxPath = _Path + "Replication" + _S + "Inbox"; _OutboxPath = _Path + "Replication" + _S + "Outbox"; _config = fastJSON.JSON.ToObject(config); if (_config == null) { _log.Error("unable to read the configuration for replication, check the config file"); return; } // read branch lastdoc counts here foreach (var b in _config.Where) { int i = -1; if (File.Exists(_Path + "Replication" + _S + b.BranchName + ".last")) i = Helper.ToInt32(File.ReadAllBytes(_Path + "Replication" + _S + b.BranchName + ".last"), 0); Directory.CreateDirectory(_Path + "Replication" + _S + "Inbox" + _S + b.BranchName); Directory.CreateDirectory(_Path + "Replication" + _S + "Outbox" + _S + b.BranchName); _branchLastDocs.Add(b.BranchName.ToLower(), i); } _server = new NetworkServer(); _server.Start(_config.ReplicationPort, processpayload); } public void Shutdown() { WriteBranchCounters(); // shutdown every thing _server.Stop(); } private void WriteBranchCounters() { // write branch counts etc. to disk foreach (var b in _branchLastDocs) { File.WriteAllBytes(_Path + "Replication" + _S + b.Key + ".last", Helper.GetBytes(b.Value, false)); _log.Debug("last counter for branch : " + b.Key + " = " + b.Value); } } private object processpayload(object data) { ReplicationPacket p = (ReplicationPacket)data; if (Authenticate(p) == false) return new ReturnPacket(false, "Authentication failed"); ReturnPacket ret = new ReturnPacket(true); try { switch (p.command) { case "getbranchconfig": ret.OK = true; ret.Data = GetBranchConfig(p.branchname); break; case "getpackageforbranch": ret.OK = true; ReplicationPacket pack = GetPackageForBranch(p); ret.Data = pack; break; case "packageforhq": ret.OK = PackageForHQ(p); break; case "hqpackageok": ret.OK = true; File.Delete(_OutboxPath + _S + p.branchname + _S + p.filename); // set last rec on hq _branchLastDocs.Add(p.branchname.ToLower(), p.lastrecord); WriteBranchCounters(); break; } } catch (Exception ex) { ret.OK = false; _log.Error(ex); } return ret; } private ClientWhatWhenConfig GetBranchConfig(string branchname) { WhatItem ret = _config.What.Find((WhatItem w) => { return w.Name.ToLower() == branchname.ToLower(); }); if (ret == null) ret = _config.What.Find((WhatItem w) => { return w.Name.ToLower() == "default"; }); ClientWhatWhenConfig c = new ClientWhatWhenConfig(); c.what = ret; var where = _config.Where.Find(w => { return w.BranchName.ToLower() == branchname.ToLower(); }); if (where != null) c.whencron = where.When; else c.whencron = "* * * * *"; return c; } private bool PackageForHQ(ReplicationPacket p) { uint hash = Helper.MurMur.Hash((byte[])p.data); if (hash != p.datahash) return false; // save file to \replication\inbox\branchname Directory.CreateDirectory(_InboxPath + _S + p.branchname); string fn = _InboxPath + _S + p.branchname + _S + p.filename; _log.Debug("package recieved from : " + p.branchname); _log.Debug("package name : " + p.filename); _log.Debug("package size : " + (p.data as byte[]).Length.ToString("#,0")); File.WriteAllBytes(fn, (byte[])p.data); return true; } private ReplicationPacket GetPackageForBranch(ReplicationPacket packet) { int last = _branchLastDocs[packet.branchname.ToLower()]; // skip retry for the same package if (packet.lastrecord >= _branchLastDocs[packet.branchname.ToLower()]) { string fn = CreatePackageForSend(packet, out last); ReplicationPacket p = new ReplicationPacket(); p.filename = Path.GetFileName(fn); p.data = File.ReadAllBytes(fn); p.datahash = Helper.MurMur.Hash((byte[])p.data); p.lastrecord = last; return p; } else return null; } private bool Authenticate(ReplicationPacket p) { uint pwd = uint.Parse(p.passwordhash); bool auth = false; foreach (var w in _config.Where) { uint hash = Helper.MurMur.Hash(Helper.GetBytes(w.BranchName + "|" + w.Password)); if (hash == pwd) auth = true; } if (auth == false) _log.Debug("Authentication failed for '" + p.branchname + "' hash = " + p.passwordhash); return auth; } private string CreatePackageForSend(ReplicationPacket packet, out int last) { int maxc = INTERNALLIMIT; WhatItem what = GetBranchConfig(packet.branchname).what; if (what.PackageItemLimit > 0) maxc = what.PackageItemLimit; string outFolder = _OutboxPath; int packageNumber = packet.lastrecord; int i = packet.lastrecord; string filename = outFolder + _S + packet.branchname + _S + packageNumber.ToString("0000000000") + ".mgdat"; if (i < _docs.RecordCount()) { StorageFile package = new StorageFile(filename, SF_FORMAT.JSON, true); while (maxc > 0) { var meta = _docs.GetMeta(i); if (meta == null) break; if (meta.isReplicated == false && MatchType(meta.typename, what)) { if (meta.isDeleted == false || what.PropogateHQDeletes) { object obj = _docs.GetObject(i, out meta); package.WriteObject(meta.key, obj); maxc--; } } i++; } package.Shutdown(); packageNumber++; // compress the file using (FileStream read = File.OpenRead(filename)) using (FileStream outp = File.Create(filename + ".gz")) CompressForBackup(read, outp); // delete uncompressed file File.Delete(filename); } last = i; return filename + ".gz"; } private bool MatchType(string typename, WhatItem what) { // match type filter foreach (var i in what.HQ2Btypes) { // do wildcard search Regex reg = new Regex("^" + i.Replace("*", ".*").Replace("?", "."), RegexOptions.IgnoreCase); if (reg.IsMatch(typename)) return true; } return false; } private static void CompressForBackup(Stream source, Stream destination) { using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress)) PumpDataForBackup(source, gz); } private static void PumpDataForBackup(Stream input, Stream output) { byte[] bytes = new byte[4096 * 2]; int n; while ((n = input.Read(bytes, 0, bytes.Length)) != 0) output.Write(bytes, 0, n); } } } ================================================ FILE: RaptorDB/Replication/msg.txt ================================================ The following error occurred and the json document is below, you can skip this document if you wish by incrementing the %c% file : %ex% ------------------------------------------------------------------------------ The json document is : %js% ================================================ FILE: RaptorDB/Storage/KeyStore.cs ================================================ using System; using System.Collections.Generic; using System.Text; using System.IO; using RaptorDB.Common; namespace RaptorDB { #region [ KeyStoreString ] internal class KeyStoreString : IDisposable { public KeyStoreString(string filename, bool caseSensitve) { _db = KeyStore.Open(filename, true); _caseSensitive = caseSensitve; } bool _caseSensitive = false; KeyStore _db; public void Set(string key, string val) { Set(key, fastJSON.Reflection.UnicodeGetBytes(val)); } public void Set(string key, byte[] val) { string str = (_caseSensitive ? key : key.ToLower()); byte[] bkey = fastJSON.Reflection.UnicodeGetBytes(str); int hc = (int)Helper.MurMur.Hash(bkey); MemoryStream ms = new MemoryStream(); ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4); ms.Write(bkey, 0, bkey.Length); ms.Write(val, 0, val.Length); _db.SetBytes(hc, ms.ToArray()); } public bool Get(string key, out string val) { val = null; byte[] bval; bool b = Get(key, out bval); if (b) { val = fastJSON.Reflection.UnicodeGetString(bval); } return b; } public bool Get(string key, out byte[] val) { string str = (_caseSensitive ? key : key.ToLower()); val = null; byte[] bkey = fastJSON.Reflection.UnicodeGetBytes(str); int hc = (int)Helper.MurMur.Hash(bkey); if (_db.GetBytes(hc, out val)) { // unpack data byte[] g = null; if (UnpackData(val, out val, out g)) { if (Helper.CompareMemCmp(bkey, g) != 0) { // if data not equal check duplicates (hash conflict) List ints = new List(_db.GetDuplicates(hc)); ints.Reverse(); foreach (int i in ints) { byte[] bb = _db.FetchRecordBytes(i); if (UnpackData(bb, out val, out g)) { if (Helper.CompareMemCmp(bkey, g) == 0) return true; } } return false; } return true; } } return false; } public int Count() { return (int)_db.Count(); } public int RecordCount() { return (int)_db.RecordCount(); } public void SaveIndex() { _db.SaveIndex(); } public void Shutdown() { _db.Shutdown(); } public void Dispose() { _db.Shutdown(); } private bool UnpackData(byte[] buffer, out byte[] val, out byte[] key) { int len = Helper.ToInt32(buffer, 0, false); key = new byte[len]; Buffer.BlockCopy(buffer, 4, key, 0, len); val = new byte[buffer.Length - 4 - len]; Buffer.BlockCopy(buffer, 4 + len, val, 0, buffer.Length - 4 - len); return true; } public string ReadData(int recnumber) { byte[] val; byte[] key; byte[] b = _db.FetchRecordBytes(recnumber); if (UnpackData(b, out val, out key)) { return fastJSON.Reflection.UnicodeGetString(val); } return ""; } internal void FreeMemory() { _db.FreeMemory(); } } #endregion #region [ KeyStoreGuid removed ] //internal class KeyStoreGuid : IDisposable //, IDocStorage //{ // public KeyStoreGuid(string filename) // { // _db = KeyStore.Open(filename, true); // } // KeyStore _db; // public void Set(Guid key, string val) // { // Set(key, Encoding.Unicode.GetBytes(val)); // } // public int Set(Guid key, byte[] val) // { // byte[] bkey = key.ToByteArray(); // int hc = (int)Helper.MurMur.Hash(bkey); // MemoryStream ms = new MemoryStream(); // ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4); // ms.Write(bkey, 0, bkey.Length); // ms.Write(val, 0, val.Length); // return _db.SetBytes(hc, ms.ToArray()); // } // public bool Get(Guid key, out string val) // { // val = null; // byte[] bval; // bool b = Get(key, out bval); // if (b) // { // val = Encoding.Unicode.GetString(bval); // } // return b; // } // public bool Get(Guid key, out byte[] val) // { // val = null; // byte[] bkey = key.ToByteArray(); // int hc = (int)Helper.MurMur.Hash(bkey); // if (_db.Get(hc, out val)) // { // // unpack data // byte[] g = null; // if (UnpackData(val, out val, out g)) // { // if (Helper.CompareMemCmp(bkey, g) != 0) // { // // if data not equal check duplicates (hash conflict) // List ints = new List(_db.GetDuplicates(hc)); // ints.Reverse(); // foreach (int i in ints) // { // byte[] bb = _db.FetchRecordBytes(i); // if (UnpackData(bb, out val, out g)) // { // if (Helper.CompareMemCmp(bkey, g) == 0) // return true; // } // } // return false; // } // return true; // } // } // return false; // } // public void SaveIndex() // { // _db.SaveIndex(); // } // public void Shutdown() // { // _db.Shutdown(); // } // public void Dispose() // { // _db.Shutdown(); // } // public byte[] FetchRecordBytes(int record) // { // return _db.FetchRecordBytes(record); // } // public int Count() // { // return (int)_db.Count(); // } // public int RecordCount() // { // return (int)_db.RecordCount(); // } // private bool UnpackData(byte[] buffer, out byte[] val, out byte[] key) // { // int len = Helper.ToInt32(buffer, 0, false); // key = new byte[len]; // Buffer.BlockCopy(buffer, 4, key, 0, len); // val = new byte[buffer.Length - 4 - len]; // Buffer.BlockCopy(buffer, 4 + len, val, 0, buffer.Length - 4 - len); // return true; // } // internal byte[] Get(int recnumber, out Guid docid) // { // bool isdeleted = false; // return Get(recnumber, out docid, out isdeleted); // } // public bool RemoveKey(Guid key) // { // byte[] bkey = key.ToByteArray(); // int hc = (int)Helper.MurMur.Hash(bkey); // MemoryStream ms = new MemoryStream(); // ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4); // ms.Write(bkey, 0, bkey.Length); // return _db.Delete(hc, ms.ToArray()); // } // public byte[] Get(int recnumber, out Guid docid, out bool isdeleted) // { // docid = Guid.Empty; // byte[] buffer = _db.FetchRecordBytes(recnumber, out isdeleted); // if (buffer == null) return null; // if (buffer.Length == 0) return null; // byte[] key; // byte[] val; // // unpack data // UnpackData(buffer, out val, out key); // docid = new Guid(key); // return val; // } // internal int CopyTo(StorageFile backup, int start) // { // return _db.CopyTo(backup, start); // } //} #endregion internal class KeyStore : IDisposable, IDocStorage where T : IComparable { public KeyStore(string Filename, byte MaxKeySize, bool AllowDuplicateKeys) { Initialize(Filename, MaxKeySize, AllowDuplicateKeys); } public KeyStore(string Filename, bool AllowDuplicateKeys) { Initialize(Filename, Global.DefaultStringKeySize, AllowDuplicateKeys); } private ILog log = LogManager.GetLogger(typeof(KeyStore)); private string _Path = ""; private string _FileName = ""; private byte _MaxKeySize; private StorageFile _archive; private MGIndex _index; private string _datExtension = ".mgdat"; private string _idxExtension = ".mgidx"; private IGetBytes _T = null; private System.Timers.Timer _savetimer; private BoolIndex _deleted; public static KeyStore Open(string Filename, bool AllowDuplicateKeys) { return new KeyStore(Filename, AllowDuplicateKeys); } public static KeyStore Open(string Filename, byte MaxKeySize, bool AllowDuplicateKeys) { return new KeyStore(Filename, MaxKeySize, AllowDuplicateKeys); } object _savelock = new object(); public void SaveIndex() { if (_index == null) return; lock (_savelock) { log.Debug("saving to disk"); _index.SaveIndex(); _deleted.SaveIndex(); log.Debug("index saved"); } } public IEnumerable GetDuplicates(T key) { // get duplicates from index return _index.GetDuplicates(key); } public byte[] FetchRecordBytes(int record) { return _archive.ReadBytes(record); } public long Count() { int c = _archive.Count(); return c - _deleted.GetBits().CountOnes() * 2; } public bool Get(T key, out string val) { byte[] b = null; val = ""; bool ret = GetBytes(key, out b); if (ret) { if (b != null) val = fastJSON.Reflection.UnicodeGetString(b); else val = ""; } return ret; } public bool GetObject(T key, out object val) { int off; val = null; if (_index.Get(key, out off)) { val = _archive.ReadObject(off); return true; } return false; } public bool GetBytes(T key, out byte[] val) { int off; val = null; // search index if (_index.Get(key, out off)) { val = _archive.ReadBytes(off); return true; } return false; } public int SetString(T key, string data) { return SetBytes(key, fastJSON.Reflection.UnicodeGetBytes(data)); } public int SetObject(T key, object doc) { int recno = -1; // save to storage recno = (int) _archive.WriteObject(key, doc); // save to index _index.Set(key, recno); return recno; } public int SetBytes(T key, byte[] data) { int recno = -1; // save to storage recno = (int)_archive.WriteData(key, data); // save to index _index.Set(key, recno); return recno; } private object _shutdownlock = new object(); public void Shutdown() { lock (_shutdownlock) { if (_index != null) log.Debug("Shutting down"); else return; _savetimer.Enabled = false; SaveIndex(); SaveLastRecord(); if (_deleted != null) _deleted.Shutdown(); if (_index != null) _index.Shutdown(); if (_archive != null) _archive.Shutdown(); _index = null; _archive = null; _deleted = null; //log.Debug("Shutting down log"); //LogManager.Shutdown(); } } public void Dispose() { Shutdown(); } #region [ P R I V A T E M E T H O D S ] private void SaveLastRecord() { // save the last record number in the index file _index.SaveLastRecordNumber(_archive.Count()); } private void Initialize(string filename, byte maxkeysize, bool AllowDuplicateKeys) { _MaxKeySize = RDBDataType.GetByteSize(maxkeysize); _T = RDBDataType.ByteHandler(); _Path = Path.GetDirectoryName(filename); Directory.CreateDirectory(_Path); _FileName = Path.GetFileNameWithoutExtension(filename); string db = _Path + Path.DirectorySeparatorChar + _FileName + _datExtension; string idx = _Path + Path.DirectorySeparatorChar + _FileName + _idxExtension; //LogManager.Configure(_Path + Path.DirectorySeparatorChar + _FileName + ".txt", 500, false); _index = new MGIndex(_Path, _FileName + _idxExtension, _MaxKeySize, /*Global.PageItemCount,*/ AllowDuplicateKeys); if (Global.SaveAsBinaryJSON) _archive = new StorageFile(db, SF_FORMAT.BSON, false); else _archive = new StorageFile(db, SF_FORMAT.JSON, false); _deleted = new BoolIndex(_Path, _FileName , "_deleted.idx"); log.Debug("Current Count = " + RecordCount().ToString("#,0")); CheckIndexState(); log.Debug("Starting save timer"); _savetimer = new System.Timers.Timer(); _savetimer.Elapsed += new System.Timers.ElapsedEventHandler(_savetimer_Elapsed); _savetimer.Interval = Global.SaveIndexToDiskTimerSeconds * 1000; _savetimer.AutoReset = true; _savetimer.Start(); } private void CheckIndexState() { log.Debug("Checking Index state..."); int last = _index.GetLastIndexedRecordNumber(); int count = _archive.Count(); if (last < count) { log.Debug("Rebuilding index..."); log.Debug(" last index count = " + last); log.Debug(" data items count = " + count); // check last index record and archive record // rebuild index if needed for (int i = last; i < count; i++) { bool deleted = false; T key = _archive.GetKey(i, out deleted); if (deleted == false) _index.Set(key, i); else _index.RemoveKey(key); if (i % 100000 == 0) log.Debug("100,000 items re-indexed"); } log.Debug("Rebuild index done."); } } void _savetimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { SaveIndex(); } #endregion public int RecordCount() { return _archive.Count(); } public int[] GetHistory(T key) { List a = new List(); foreach (int i in GetDuplicates(key)) { a.Add(i); } return a.ToArray(); } internal byte[] FetchRecordBytes(int record, out bool isdeleted) { StorageItem meta; byte[] b = _archive.ReadBytes(record, out meta); isdeleted = meta.isDeleted; return b; } internal bool Delete(T id) { // write a delete record int rec = (int)_archive.Delete(id); _deleted.Set(true, rec); return _index.RemoveKey(id); } internal bool DeleteReplicated(T id) { // write a delete record for replicated object int rec = (int)_archive.DeleteReplicated(id); _deleted.Set(true, rec); return _index.RemoveKey(id); } internal int CopyTo(StorageFile storagefile, long startrecord) { return _archive.CopyTo(storagefile, startrecord); } public byte[] GetBytes(int rowid, out StorageItem meta) { return _archive.ReadBytes(rowid, out meta); } internal void FreeMemory() { _index.FreeMemory(); } public object GetObject(int rowid, out StorageItem meta) { return _archive.ReadObject(rowid, out meta); } public StorageItem GetMeta(int rowid) { return _archive.ReadMeta(rowid); } internal int SetReplicationObject(T key, object doc) { int recno = -1; // save to storage recno = (int) _archive.WriteReplicationObject(key, doc); // save to index _index.Set(key, recno); return recno; } } } ================================================ FILE: RaptorDB/Storage/KeyStoreHF.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace RaptorDB { // high frequency key value store public class KeyStoreHF : IKeyStoreHF { internal class AllocationBlock { public string key; public byte keylen; public int datalength; public bool isCompressed; public bool isBinaryJSON; public bool deleteKey; public List Blocks = new List(); public int blocknumber; } MGIndex _keys; StorageFileHF _datastore; object _lock = new object(); ushort _BlockSize = 2048; private const int _KILOBYTE = 1024; ILog _log = LogManager.GetLogger(typeof(KeyStoreHF)); byte[] _blockheader = new byte[]{ 0,0,0,0, // 0 block # (used for validate block reads and rebuild) 0,0,0,0, // 4 next block # 0, // 8 flags bits 0:iscompressed 1:isbinary 2:deletekey 0,0,0,0, // 9 data length (compute alloc blocks needed) 0, // 13 key length 0, // 14 key type 0=guid 1=string }; private string _Path = ""; private string _S = Path.DirectorySeparatorChar.ToString(); private bool _isDirty = false; private string _dirtyFilename = "temp.$"; public KeyStoreHF(string folder) { _Path = folder; Directory.CreateDirectory(_Path); if (_Path.EndsWith(_S) == false) _Path += _S; if (File.Exists(_Path + _dirtyFilename)) { _log.Error("Last shutdown failed, rebuilding data files..."); RebuildDataFiles(); } _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize); _keys = new MGIndex(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false); //_datastore.Initialize(); _BlockSize = _datastore.GetBlockSize(); } //// mgindex special storage for strings ctor -> no idx file //// use SaveData() GetData() //public KeyStoreHF(string folder, string filename) //{ // _Path = folder; // Directory.CreateDirectory(_Path); // if (_Path.EndsWith(_S) == false) _Path += _S; // _datastore = new StorageFileHF(_Path + filename, Global.HighFrequencyKVDiskBlockSize); // //_datastore.Initialize(); // _BlockSize = _datastore.GetBlockSize(); //} public int CountHF() { if (_keys != null) return _keys.Count(); else return 0; } public object GetObjectHF(string key) { lock (_lock) { int alloc; if (_keys.Get(key, out alloc)) { AllocationBlock ab = FillAllocationBlock(alloc); if (ab.deleteKey == false) { byte[] data = readblockdata(ab); return fastBinaryJSON.BJSON.ToObject(data); } } } return null; } public bool SetObjectHF(string key, object obj) { byte[] k = Helper.GetBytes(key); if (k.Length > 255) { _log.Error("Key length > 255 : " + key); throw new Exception("Key must be less than 255 characters"); //return false; } lock (_lock) { if (_isDirty == false) WriteDirtyFile(); AllocationBlock ab = null; int firstblock = 0; if (_keys.Get(key, out firstblock))// key exists already ab = FillAllocationBlock(firstblock); SaveNew(key, k, obj); if (ab != null) { // free old blocks ab.Blocks.Add(ab.blocknumber); _datastore.FreeBlocks(ab.Blocks); } return true; } } public bool DeleteKeyHF(string key) { lock (_lock) { int alloc; if (_keys.Get(key, out alloc)) { if (_isDirty == false) WriteDirtyFile(); byte[] keybytes = Helper.GetBytes(key); AllocationBlock ab = FillAllocationBlock(alloc); ab.keylen = (byte)keybytes.Length; _keys.RemoveKey(key);// remove key from index // write ab ab.deleteKey = true; ab.datalength = 0; byte[] header = CreateAllocHeader(ab, keybytes); _datastore.SeekBlock(ab.blocknumber); _datastore.WriteBlockBytes(header, 0, header.Length); // free old data blocks _datastore.FreeBlocks(ab.Blocks); return true; } } return false; } public void CompactStorageHF() { lock (_lock) { try { _log.Debug("Compacting storage file ..."); if (Directory.Exists(_Path + "temp")) Directory.Delete(_Path + "temp", true); KeyStoreHF newfile = new KeyStoreHF(_Path + "temp"); string[] keys = _keys.GetKeys().Cast().ToArray(); _log.Debug("Number of keys : " + keys.Length); foreach (var k in keys) { newfile.SetObjectHF(k, GetObjectHF(k)); } newfile.Shutdown(); _log.Debug("Compact done."); // shutdown and move files and restart here if (Directory.Exists(_Path + "old")) Directory.Delete(_Path + "old", true); Directory.CreateDirectory(_Path + "old"); _datastore.Shutdown(); _keys.Shutdown(); _log.Debug("Moving files..."); foreach (var f in Directory.GetFiles(_Path, "*.*")) File.Move(f, _Path + "old" + _S + Path.GetFileName(f)); foreach (var f in Directory.GetFiles(_Path + "temp", "*.*")) File.Move(f, _Path + Path.GetFileName(f)); Directory.Delete(_Path + "temp", true); //Directory.Delete(_Path + "old", true); // FEATURE : delete or keep? _log.Debug("Re-opening storage file"); _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize); _keys = new MGIndex(_Path, "keys.idx", 255, false); _BlockSize = _datastore.GetBlockSize(); } catch (Exception ex) { _log.Error(ex); } } } public string[] GetKeysHF() { lock (_lock) return _keys.GetKeys().Cast().ToArray(); // FEATURE : ugly and dirty !? } public bool ContainsHF(string key) { lock (_lock) { int i = 0; return _keys.Get(key, out i); } } public void Shutdown() { _datastore.Shutdown(); if (_keys != null) _keys.Shutdown(); if (File.Exists(_Path + _dirtyFilename)) File.Delete(_Path + _dirtyFilename); } internal void FreeMemory() { if (_keys != null) _keys.FreeMemory(); } #region [ private methods ] private byte[] readblockdata(AllocationBlock ab) { byte[] data = new byte[ab.datalength]; long offset = 0; int len = ab.datalength; int dbsize = _BlockSize - _blockheader.Length - ab.keylen; ab.Blocks.ForEach(x => { byte[] b = _datastore.ReadBlock(x); int c = len; if (c > dbsize) c = dbsize; Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c); offset += c; len -= c; }); if (ab.isCompressed) data = MiniLZO.Decompress(data); return data; } private object _dfile = new object(); private void WriteDirtyFile() { lock (_dfile) { _isDirty = true; if (File.Exists(_Path + _dirtyFilename) == false) File.WriteAllText(_Path + _dirtyFilename, "dirty"); } } private void SaveNew(string key, byte[] keybytes, object obj) { byte[] data; AllocationBlock ab = new AllocationBlock(); ab.key = key; ab.keylen = (byte)keybytes.Length; data = fastBinaryJSON.BJSON.ToBJSON(obj); ab.isBinaryJSON = true; if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE) { ab.isCompressed = true; data = MiniLZO.Compress(data); } ab.datalength = data.Length; int firstblock = internalSave(keybytes, data, ab); // save keys _keys.Set(key, firstblock); } private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab) { int firstblock = _datastore.GetFreeBlockNumber(); int blocknum = firstblock; byte[] header = CreateAllocHeader(ab, keybytes); int dblocksize = _BlockSize - header.Length; int offset = 0; // compute data block count int datablockcount = (data.Length / dblocksize) + 1; // save data blocks int counter = 0; int len = data.Length; while (datablockcount > 0) { datablockcount--; int next = 0; if (datablockcount > 0) next = _datastore.GetFreeBlockNumber(); Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4); // set block number Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4); // set next pointer _datastore.SeekBlock(blocknum); _datastore.WriteBlockBytes(header, 0, header.Length); int c = len; if (c > dblocksize) c = dblocksize; _datastore.WriteBlockBytes(data, offset, c); if (next > 0) blocknum = next; offset += c; len -= c; counter++; } return firstblock; } private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes) { byte[] alloc = new byte[_blockheader.Length + keybytes.Length]; if (ab.isCompressed) alloc[8] = 1; if (ab.isBinaryJSON) alloc[8] += 2; if (ab.deleteKey) alloc[8] += 4; Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4); alloc[13] = ab.keylen; alloc[14] = 1; // string keys for now Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen); return alloc; } private AllocationBlock FillAllocationBlock(int blocknumber) { AllocationBlock ab = new AllocationBlock(); ab.blocknumber = blocknumber; ab.Blocks.Add(blocknumber); byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255); int blocknumexpected = 0; int next = ParseBlockHeader(ab, b, blocknumexpected); blocknumexpected++; while (next > 0) { ab.Blocks.Add(next); b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen); next = ParseBlockHeader(ab, b, blocknumexpected); blocknumexpected++; } return ab; } private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected) { int bnum = Helper.ToInt32(b, 0); if (bnum != blocknumberexpected) { _log.Error("Block numbers does not match, looking for : " + blocknumberexpected); //throw new Exception("Block numbers does not match, looking for : " + blocknumberexpected); return -1; } if (b[14] != 1) { _log.Error("Expecting string keys only, got : " + b[14]); //throw new Exception("Expecting string keys only, got : " + b[11]); return -1; } int next = Helper.ToInt32(b, 4); if (ab.keylen == 0) { byte flags = b[8]; if ((flags & 0x01) > 0) ab.isCompressed = true; if ((flags & 0x02) > 0) ab.isBinaryJSON = true; if ((flags & 0x04) > 0) ab.deleteKey = true; ab.datalength = Helper.ToInt32(b, 9); byte keylen = b[13]; ab.keylen = keylen; ab.key = Helper.GetString(b, _blockheader.Length, keylen); } return next; } private void RebuildDataFiles() { MGIndex keys = null; try { // remove old free list if (File.Exists(_Path + "data.bmp")) File.Delete(_Path + "data.bmp"); _datastore = new StorageFileHF(_Path + "data.mghf", Global.HighFrequencyKVDiskBlockSize); _BlockSize = _datastore.GetBlockSize(); if (File.Exists(_Path + "keys.idx")) { _log.Debug("removing old keys index"); foreach (var f in Directory.GetFiles(_Path, "keys.*")) File.Delete(f); } keys = new MGIndex(_Path, "keys.idx", 255, /*Global.PageItemCount,*/ false); MGRB visited = new MGRB(); int c = _datastore.NumberofBlocks(); for (int i = 1; i < c; i++) // go through blocks skip first { if (visited.Get(i)) continue; byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255); int bnum = Helper.ToInt32(b, 0); if (bnum > 0) // check if a start block { visited.Set(i, true); _datastore.FreeBlock(i); // mark as free continue; } AllocationBlock ab = new AllocationBlock(); // start block found int blocknumexpected = 0; int next = ParseBlockHeader(ab, b, blocknumexpected); int last = 0; bool freelast = false; AllocationBlock old = null; if (ab.key == null) continue; if (keys.Get(ab.key, out last)) { old = this.FillAllocationBlock(last); freelast = true; } blocknumexpected++; bool failed = false; if (ab.deleteKey == false) { while (next > 0) // read the blocks { ab.Blocks.Add(next); b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen); next = ParseBlockHeader(ab, b, blocknumexpected); if (next == -1) // non matching block { failed = true; break; } blocknumexpected++; } } else { failed = true; keys.RemoveKey(ab.key); } // new data ok if (failed == false) { keys.Set(ab.key, i);// valid block found if (freelast && old != null)// free the old blocks _datastore.FreeBlocks(old.Blocks); } visited.Set(i, true); } // all ok delete temp.$ file if (File.Exists(_Path + _dirtyFilename)) File.Delete(_Path + _dirtyFilename); } catch (Exception ex) { _log.Error(ex); } finally { _log.Debug("Shutting down files and index"); _datastore.Shutdown(); keys.SaveIndex(); keys.Shutdown(); } } #endregion //internal void FreeBlocks(List list) //{ // lock (_lock) // _datastore.FreeBlocks(list); //} //// for .string files //internal int SaveData(string key, byte[] data) //{ // lock (_lock) // { // byte[] kb = Helper.GetBytes(key); // AllocationBlock ab = new AllocationBlock(); // ab.key = key; // ab.keylen = (byte)kb.Length; // ab.isCompressed = false; // ab.isBinaryJSON = true; // ab.datalength = data.Length; // return internalSave(kb, data, ab); // } //} //// for .string files //internal byte[] GetData(int blocknumber, out List usedblocks) //{ // lock (_lock) // { // AllocationBlock ab = FillAllocationBlock(blocknumber); // usedblocks = ab.Blocks; // byte[] data = readblockdata(ab); // return data; // } //} public int Increment(string key, int amount) { byte[] k = Helper.GetBytes(key); if (k.Length > 255) { _log.Error("Key length > 255 : " + key); throw new Exception("Key must be less than 255 characters"); //return false; } lock (_lock) { if (_isDirty == false) WriteDirtyFile(); AllocationBlock ab = null; int firstblock = 0; if (_keys.Get(key, out firstblock))// key exists already ab = FillAllocationBlock(firstblock); object obj = amount; if (ab.deleteKey == false) { byte[] data = readblockdata(ab); obj = fastBinaryJSON.BJSON.ToObject(data); // add here if (obj is int) obj = ((int)obj) + amount; else if (obj is long) obj = ((long)obj) + amount; else if (obj is decimal) obj = ((decimal)obj) + amount; else return (int)obj; } SaveNew(key, k, obj); if (ab != null) { // free old blocks ab.Blocks.Add(ab.blocknumber); _datastore.FreeBlocks(ab.Blocks); } return (int)obj; } } public int Decrement(string key, int amount) { return (int)Increment(key, -amount); } public decimal Increment(string key, decimal amount) { byte[] k = Helper.GetBytes(key); if (k.Length > 255) { _log.Error("Key length > 255 : " + key); throw new Exception("Key must be less than 255 characters"); //return false; } lock (_lock) { if (_isDirty == false) WriteDirtyFile(); AllocationBlock ab = null; int firstblock = 0; if (_keys.Get(key, out firstblock))// key exists already ab = FillAllocationBlock(firstblock); object obj = amount; if (ab.deleteKey == false) { byte[] data = readblockdata(ab); obj = fastBinaryJSON.BJSON.ToObject(data); // add here if (obj is decimal) obj = ((decimal)obj) + amount; else return (decimal)obj; } SaveNew(key, k, obj); if (ab != null) { // free old blocks ab.Blocks.Add(ab.blocknumber); _datastore.FreeBlocks(ab.Blocks); } return (decimal)obj; } } public decimal Decrement(string key, decimal amount) { return Increment(key, -amount); } } } ================================================ FILE: RaptorDB/Storage/StorageFile.cs ================================================ using System; using System.IO; using System.Text; using System.Collections.Generic; using RaptorDB.Common; using fastBinaryJSON; using fastJSON; namespace RaptorDB { internal class StorageData { public StorageItem meta; public byte[] data; } public class StorageItem { public T key; public string typename; public DateTime date = FastDateTime.Now; public bool isDeleted; public bool isReplicated; public int dataLength; public byte isCompressed; // 0 = no, 1 = MiniLZO } public interface IDocStorage { int RecordCount(); byte[] GetBytes(int rowid, out StorageItem meta); object GetObject(int rowid, out StorageItem meta); StorageItem GetMeta(int rowid); bool GetObject(T key, out object doc); } public enum SF_FORMAT { BSON, JSON } internal struct SplitFile { public long start; public long uptolength; public FileStream file; } public class StorageFile { FileStream _datawrite; FileStream _recfilewrite; FileStream _recfileread = null; FileStream _dataread = null; private string _filename = ""; private string _recfilename = ""; private long _lastRecordNum = 0; private long _lastWriteOffset = _fileheader.Length; private object _readlock = new object(); private bool _dirty = false; IGetBytes _T = null; ILog _log = LogManager.GetLogger(typeof(StorageFile)); private SF_FORMAT _saveFormat = SF_FORMAT.BSON; // **** change this if storage format changed **** internal static int _CurrentVersion = 2; //private ushort _splitMegaBytes = 0; // 0 = off //private bool _enableSplits = false; private List _files = new List(); private List _uptoindexes = new List(); // no splits in view mode private bool _viewmode = false; private SplitFile _lastsplitfile; public static byte[] _fileheader = { (byte)'M', (byte)'G', (byte)'D', (byte)'B', 0, // 4 -- storage file version number, 0 // 5 -- not used }; private static string _splitfileExtension = "00000"; private const int _KILOBYTE = 1024; // record format : // 1 type (0 = raw no meta data, 1 = bson meta, 2 = json meta) // 4 byte meta/data length, // n byte meta serialized data if exists // m byte data (if meta exists then m is in meta.dataLength) /// /// View data storage mode (no splits, bson save) /// /// public StorageFile(string filename) { _viewmode = true; _saveFormat = SF_FORMAT.BSON; // add version number _fileheader[5] = (byte)_CurrentVersion; Initialize(filename, false); } /// /// /// /// /// /// = true -> don't create mgrec files (used for backup and replication mode) public StorageFile(string filename, SF_FORMAT format, bool StorageOnlyMode) { _saveFormat = format; if (StorageOnlyMode) _viewmode = true; // no file splits // add version number _fileheader[5] = (byte)_CurrentVersion; Initialize(filename, StorageOnlyMode); } private StorageFile(string filename, bool StorageOnlyMode) { Initialize(filename, StorageOnlyMode); } private void Initialize(string filename, bool StorageOnlyMode) { _T = RDBDataType.ByteHandler(); _filename = filename; // search for mgdat00000 extensions -> split files load if (File.Exists(filename + _splitfileExtension)) { LoadSplitFiles(filename); } if (File.Exists(filename) == false) _datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); else _datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); _dataread = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); if (_datawrite.Length == 0) { // new file _datawrite.Write(_fileheader, 0, _fileheader.Length); _datawrite.Flush(); _lastWriteOffset = _fileheader.Length; } else { long i = _datawrite.Seek(0L, SeekOrigin.End); if (_files.Count == 0) _lastWriteOffset = i; else _lastWriteOffset += i; // add to the splits } if (StorageOnlyMode == false) { // load rec pointers _recfilename = filename.Substring(0, filename.LastIndexOf('.')) + ".mgrec"; if (File.Exists(_recfilename) == false) _recfilewrite = new FileStream(_recfilename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite); else _recfilewrite = new FileStream(_recfilename, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); _recfileread = new FileStream(_recfilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); _lastRecordNum = (int)(_recfilewrite.Length / 8); _recfilewrite.Seek(0L, SeekOrigin.End); } } private void LoadSplitFiles(string filename) { _log.Debug("Loading split files..."); _lastWriteOffset = 0; for (int i = 0; ; i++) { string _filename = filename + i.ToString(_splitfileExtension); if (File.Exists(_filename) == false) break; FileStream file = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); SplitFile sf = new SplitFile(); sf.start = _lastWriteOffset; _lastWriteOffset += file.Length; sf.file = file; sf.uptolength = _lastWriteOffset; _files.Add(sf); _uptoindexes.Add(sf.uptolength); } _lastsplitfile = _files[_files.Count - 1]; _log.Debug("Number of split files = " + _files.Count); } public static int GetStorageFileHeaderVersion(string filename) { string fn = filename + _splitfileExtension; // if split files -> load the header from the first file -> mgdat00000 if (File.Exists(fn) == false) fn = filename; // else use the mgdat file if (File.Exists(fn)) { var fs = new FileStream(fn, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); fs.Seek(0L, SeekOrigin.Begin); byte[] b = new byte[_fileheader.Length]; fs.Read(b, 0, _fileheader.Length); fs.Close(); return b[5]; } return _CurrentVersion; } public int Count() { return (int)_lastRecordNum;// (int)(_recfilewrite.Length >> 3); } public long WriteRawData(byte[] b) { return internalWriteData(null, b, true); } public long Delete(T key) { StorageItem meta = new StorageItem(); meta.key = key; meta.isDeleted = true; return internalWriteData(meta, null, false); } public long DeleteReplicated(T key) { StorageItem meta = new StorageItem(); meta.key = key; meta.isReplicated = true; meta.isDeleted = true; return internalWriteData(meta, null, false); } public long WriteObject(T key, object obj) { StorageItem meta = new StorageItem(); meta.key = key; meta.typename = Reflection.Instance.GetTypeAssemblyName(obj.GetType()); byte[] data; if (_saveFormat == SF_FORMAT.BSON) data = BJSON.ToBJSON(obj); else data = Helper.GetBytes(JSON.ToJSON(obj)); if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE) { meta.isCompressed = 1; data = MiniLZO.Compress(data); //MiniLZO } return internalWriteData(meta, data, false); } public long WriteReplicationObject(T key, object obj) { StorageItem meta = new StorageItem(); meta.key = key; meta.isReplicated = true; meta.typename = Reflection.Instance.GetTypeAssemblyName(obj.GetType()); byte[] data; if (_saveFormat == SF_FORMAT.BSON) data = BJSON.ToBJSON(obj); else data = Helper.GetBytes(JSON.ToJSON(obj)); if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE) { meta.isCompressed = 1; data = MiniLZO.Compress(data); } return internalWriteData(meta, data, false); } public long WriteData(T key, byte[] data) { StorageItem meta = new StorageItem(); meta.key = key; if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE) { meta.isCompressed = 1; data = MiniLZO.Compress(data); } return internalWriteData(meta, data, false); } public byte[] ReadBytes(long recnum) { StorageItem meta; return ReadBytes(recnum, out meta); } public object ReadObject(long recnum) { StorageItem meta = null; return ReadObject(recnum, out meta); } public object ReadObject(long recnum, out StorageItem meta) { byte[] b = ReadBytes(recnum, out meta); if (b == null) return null; if (b[0] < 32) return BJSON.ToObject(b); else return JSON.ToObject(Encoding.ASCII.GetString(b)); } /// /// used for views only /// /// /// public byte[] ViewReadRawBytes(long recnum) { // views can't be split if (recnum >= _lastRecordNum) return null; lock (_readlock) { long offset = ComputeOffset(recnum); _dataread.Seek(offset, System.IO.SeekOrigin.Begin); byte[] hdr = new byte[5]; // read header _dataread.Read(hdr, 0, 5); // meta length int len = Helper.ToInt32(hdr, 1); int type = hdr[0]; if (type == 0) { byte[] data = new byte[len]; _dataread.Read(data, 0, len); return data; } return null; } } public void Shutdown() { if (_files.Count > 0) _files.ForEach(s => FlushClose(s.file)); FlushClose(_dataread); FlushClose(_recfileread); FlushClose(_recfilewrite); FlushClose(_datawrite); _dataread = null; _recfileread = null; _recfilewrite = null; _datawrite = null; } public static StorageFile ReadForward(string filename) { StorageFile sf = new StorageFile(filename, true); return sf; } public StorageItem ReadMeta(long rowid) { if (rowid >= _lastRecordNum) return null; lock (_readlock) { int metalen = 0; long off = ComputeOffset(rowid); FileStream fs = GetReadFileStreamWithSeek(off); StorageItem meta = ReadMetaData(fs, out metalen); return meta; } } #region [ private / internal ] private long internalWriteData(StorageItem meta, byte[] data, bool raw) { lock (_readlock) { _dirty = true; // seek end of file long offset = _lastWriteOffset; if (_viewmode == false && Global.SplitStorageFilesMegaBytes > 0) { // current file size > _splitMegaBytes --> new file if (offset > (long)Global.SplitStorageFilesMegaBytes * 1024 * 1024) CreateNewStorageFile(); } if (raw == false) { if (data != null) meta.dataLength = data.Length; byte[] metabytes = BJSON.ToBJSON(meta, new BJSONParameters { UseExtensions = false, UseTypedArrays = false }); // write header info _datawrite.Write(new byte[] { 1 }, 0, 1); // FEATURE : add json here, write bson for now _datawrite.Write(Helper.GetBytes(metabytes.Length, false), 0, 4); _datawrite.Write(metabytes, 0, metabytes.Length); // update pointer _lastWriteOffset += metabytes.Length + 5; } else { // write header info _datawrite.Write(new byte[] { 0 }, 0, 1); // write raw _datawrite.Write(Helper.GetBytes(data.Length, false), 0, 4); // update pointer _lastWriteOffset += 5; } if (data != null) { // write data block _datawrite.Write(data, 0, data.Length); _lastWriteOffset += data.Length; } // return starting offset -> recno long recno = _lastRecordNum++; if (_recfilewrite != null) _recfilewrite.Write(Helper.GetBytes(offset, false), 0, 8); if (Global.FlushStorageFileImmediately) { _datawrite.Flush(); if (_recfilewrite != null) _recfilewrite.Flush(); } return recno; } } private void CreateNewStorageFile() { _log.Debug("Split limit reached = " + _datawrite.Length); int i = _files.Count; // close files FlushClose(_datawrite); FlushClose(_dataread); long start = 0; if (i > 0) start = _lastsplitfile.uptolength; // last file offset // rename mgdat to mgdat0000n File.Move(_filename, _filename + i.ToString(_splitfileExtension)); FileStream file = new FileStream(_filename + i.ToString(_splitfileExtension), FileMode.Open, FileAccess.Read, FileShare.ReadWrite); SplitFile sf = new SplitFile(); sf.start = start; sf.uptolength = _lastWriteOffset; sf.file = file; _files.Add(sf); _uptoindexes.Add(sf.uptolength); _lastsplitfile = sf; // new mgdat file _datawrite = new FileStream(_filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); _dataread = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); _log.Debug("New storage file created, count = " + _files.Count); } internal byte[] ReadBytes(long recnum, out StorageItem meta) { meta = null; if (recnum >= _lastRecordNum) return null; lock (_readlock) { long off = ComputeOffset(recnum); FileStream fs = GetReadFileStreamWithSeek(off); byte[] data = internalReadBytes(fs, out meta); if (meta.isCompressed > 0) data = MiniLZO.Decompress(data); return data; } } private long ComputeOffset(long recnum) { if (_dirty) { _datawrite.Flush(); _recfilewrite.Flush(); } long off = recnum << 3;// *8L; byte[] b = new byte[8]; _recfileread.Seek(off, SeekOrigin.Begin); _recfileread.Read(b, 0, 8); off = Helper.ToInt64(b, 0); if (off == 0)// kludge off = 6; return off; } private byte[] internalReadBytes(FileStream fs, out StorageItem meta) { int metalen = 0; meta = ReadMetaData(fs, out metalen); if (meta != null) { if (meta.isDeleted == false) { byte[] data = new byte[meta.dataLength]; fs.Read(data, 0, meta.dataLength); return data; } } else { byte[] data = new byte[metalen]; fs.Read(data, 0, metalen); return data; } return null; } private StorageItem ReadMetaData(FileStream fs, out int metasize) { byte[] hdr = new byte[5]; // read header fs.Read(hdr, 0, 5); // meta length int len = Helper.ToInt32(hdr, 1); int type = hdr[0]; if (type > 0) { metasize = len + 5; hdr = new byte[len]; fs.Read(hdr, 0, len); StorageItem meta; if (type == 1) meta = BJSON.ToObject>(hdr); else { string str = Helper.GetString(hdr, 0, (short)hdr.Length); meta = JSON.ToObject>(str); } return meta; } else { metasize = len; return null; } } private void FlushClose(FileStream st) { if (st != null) { st.Flush(true); st.Close(); } } internal T GetKey(long recnum, out bool deleted) { lock (_readlock) { deleted = false; long off = ComputeOffset(recnum); FileStream fs = GetReadFileStreamWithSeek(off); int metalen = 0; StorageItem meta = ReadMetaData(fs, out metalen); deleted = meta.isDeleted; return meta.key; } } internal int CopyTo(StorageFile storageFile, long startrecord) { FileStream fs; bool inthefiles = false; // copy data here lock (_readlock) { long off = ComputeOffset(startrecord); fs = GetReadFileStreamWithSeek(off); if (fs != _dataread) inthefiles = true; Pump(fs, storageFile._datawrite); } // pump the remainder of the files also if (inthefiles && _files.Count > 0) { long off = ComputeOffset(startrecord); int i = binarysearch(off); i++; // next file stream for (int j = i; j < _files.Count; j++) { lock (_readlock) { fs = _files[j].file; fs.Seek(0L, SeekOrigin.Begin); Pump(fs, storageFile._datawrite); } } // pump the current mgdat lock (_readlock) { _dataread.Seek(0L, SeekOrigin.Begin); Pump(_dataread, storageFile._datawrite); } } return (int)_lastRecordNum; } private static void Pump(Stream input, Stream output) { byte[] bytes = new byte[4096 * 2]; int n; while ((n = input.Read(bytes, 0, bytes.Length)) != 0) output.Write(bytes, 0, n); } internal IEnumerable> ReadOnlyEnumerate() { // MGREC files may not exist //// the total number of records //long count = _recfileread.Length >> 3; //for (long i = 0; i < count; i++) //{ // StorageItem meta; // byte[] data = ReadBytes(i, out meta); // StorageData sd = new StorageData(); // sd.meta = meta; // if (meta.dataLength > 0) // sd.data = data; // yield return sd; //} long offset = _fileheader.Length;// start; // skip header long size = _dataread.Length; while (offset < size) { StorageData sd = new StorageData(); lock (_readlock) { _dataread.Seek(offset, SeekOrigin.Begin); int metalen = 0; StorageItem meta = ReadMetaData(_dataread, out metalen); offset += metalen; sd.meta = meta; if (meta.dataLength > 0) { byte[] data = new byte[meta.dataLength]; _dataread.Read(data, 0, meta.dataLength); sd.data = data; } offset += meta.dataLength; } yield return sd; } } private FileStream GetReadFileStreamWithSeek(long offset) { long fileoffset = offset; // search split _files for offset and compute fileoffset in the file if (_files.Count > 0) // we have splits { if (offset < _lastsplitfile.uptolength) // offset is in the list { int i = binarysearch(offset); var f = _files[i]; fileoffset -= f.start; // offset in the file f.file.Seek(fileoffset, SeekOrigin.Begin); return f.file; } else fileoffset -= _lastsplitfile.uptolength; // offset in the mgdat file } // seek to position in file _dataread.Seek(fileoffset, SeekOrigin.Begin); return _dataread; } private int binarysearch(long offset) { //// binary search int low = 0; int high = _files.Count - 1; int midpoint = 0; int lastlower = 0; while (low <= high) { midpoint = low + (high - low) / 2; long k = _uptoindexes[midpoint]; // check to see if value is equal to item in array if (offset == k) return midpoint + 1; else if (offset < k) { high = midpoint - 1; lastlower = midpoint; } else low = midpoint + 1; } return lastlower; } #endregion } } ================================================ FILE: RaptorDB/Storage/StorageFileHF.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.IO; using System.Threading; namespace RaptorDB { // high frequency storage file with overwrite old values internal class StorageFileHF { private FileStream _datawriteorg; private BufferedStream _datawrite; private MGRB _freeList = new MGRB(); private string _filename = ""; private object _readlock = new object(); ILog _log = LogManager.GetLogger(typeof(StorageFileHF)); // **** change this if storage format changed **** private static int _CurrentVersion = 1; private int _lastBlockNumber = -1; private ushort _BLOCKSIZE = 4096; private string _Path = ""; private string _S = Path.DirectorySeparatorChar.ToString(); public static byte[] _fileheader = { (byte)'M', (byte)'G', (byte)'H', (byte)'F', 0, // 4 -- storage file version number, 0,2, // 5,6 -- block size ushort low, hi 1 // 7 -- key type 0 = guid, 1 = string }; //private SafeDictionary _masterblock = new SafeDictionary(); public StorageFileHF(string filename, ushort blocksize) { _Path = Path.GetDirectoryName(filename); if (_Path.EndsWith(_S) == false) _Path += _S; _filename = Path.GetFileNameWithoutExtension(filename); Initialize(filename, blocksize); } public void Shutdown() { WriteFreeListBMPFile(); _datawrite.Flush(); FlushClose(_datawriteorg); _datawrite = null; } public ushort GetBlockSize() { return _BLOCKSIZE; } internal void FreeBlocks(List list) { list.ForEach(x => _freeList.Set(x, true)); } internal byte[] ReadBlock(int blocknumber) { SeekBlock(blocknumber); byte[] data = new byte[_BLOCKSIZE]; _datawrite.Read(data, 0, _BLOCKSIZE); return data; } internal byte[] ReadBlockBytes(int blocknumber, int bytes) { SeekBlock(blocknumber); byte[] data = new byte[bytes]; _datawrite.Read(data, 0, bytes); return data; } internal int GetFreeBlockNumber() { // get the first free block or append to the end if (_freeList.CountOnes() > 0) { int i = _freeList.GetFirst(); _freeList.Set(i, false); return i; } else return Interlocked.Increment(ref _lastBlockNumber); } private void InitializeFreeList() { if (_lastBlockNumber < 0) { // write master block _datawrite.Write(new byte[_BLOCKSIZE], 0, _BLOCKSIZE); _lastBlockNumber = 1; //_masterblock.Add("freelist", -1); } else { _freeList = new MGRB(); // read master block data var b = ReadBlock(0); if (b[0] == (byte)'F' && b[1] == (byte)'L') { // get free block num and size int block = Helper.ToInt32(b, 2); int len = Helper.ToInt32(b, 2 + 4); int freeblock = block; b = new byte[len]; var offset = 0; bool failed = false; // read blocks upto size from block num SeekBlock(++block); while (len > 0) { // check header var bb = ReadBlock(block); if (bb[0] != (byte)'F' || bb[1] != (byte)'L') { // throw exception?? _log.Error("Free list header does not match : " + _filename); failed = true; break; } int c = len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len; Buffer.BlockCopy(bb, 2, b, offset, c); len -= c; offset += c; } if (failed == false) { // read freelist from master block from end of file var o = fastBinaryJSON.BJSON.ToObject(b); _freeList.Deserialize(o); // truncate end of file freelist blocks if lastblock < file size if (_datawrite.Length > _lastBlockNumber * _BLOCKSIZE) _datawrite.SetLength(_lastBlockNumber * _BLOCKSIZE); } _lastBlockNumber = freeblock; } } } internal void SeekBlock(int blocknumber) { long offset = _fileheader.Length + (long)blocknumber * _BLOCKSIZE; // wiil seek past the end of file on fs.Write will zero the difference _datawrite.Seek(offset, SeekOrigin.Begin); } internal void WriteBlockBytes(byte[] data, int start, int len) { _datawrite.Write(data, start, len); } #region [ private / internal ] private void WriteFreeListBMPFile() { // write freelist to end of blocks and update master block if (_freeList != null) { _freeList.Optimize(); var o = _freeList.Serialize(); var b = fastBinaryJSON.BJSON.ToBJSON(o, new fastBinaryJSON.BJSONParameters { UseExtensions = false }); var len = b.Length; var offset = 0; // write master block SeekBlock(0); //_lastBlockNumber++; WriteBlockBytes(new byte[] { (byte)'F', (byte)'L' }, 0, 2); WriteBlockBytes(Helper.GetBytes(_lastBlockNumber, false), 0, 4); WriteBlockBytes(Helper.GetBytes(len, false), 0, 4); // seek to end of file SeekBlock(_lastBlockNumber+1); while (len > 0) { WriteBlockBytes(new byte[] { (byte)'F', (byte)'L' }, 0, 2); WriteBlockBytes(b, offset, len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len); len -= (_BLOCKSIZE - 2); } } } private void Initialize(string filename, ushort blocksize) { if (File.Exists(filename) == false) _datawriteorg = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); else _datawriteorg = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); _datawrite = new BufferedStream(_datawriteorg); if (_datawrite.Length == 0) { CreateFileHeader(blocksize); // new file _datawrite.Write(_fileheader, 0, _fileheader.Length); _datawrite.Flush(); } else { int filever = ReadFileHeader(); if (filever < _CurrentVersion) { // fixx : upgrade storage file here } _lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE); //_lastBlockNumber++; } InitializeFreeList(); } private int ReadFileHeader() { // set _blockize _datawrite.Seek(0L, SeekOrigin.Begin); byte[] hdr = new byte[_fileheader.Length]; _datawrite.Read(hdr, 0, _fileheader.Length); _BLOCKSIZE = 0; _BLOCKSIZE = (ushort)(hdr[5] + hdr[6] << 8); return hdr[4]; } private void CreateFileHeader(int blocksize) { // add version number _fileheader[4] = (byte)_CurrentVersion; // block size _fileheader[5] = (byte)(blocksize & 0xff); _fileheader[6] = (byte)(blocksize >> 8); _BLOCKSIZE = (ushort)blocksize; } private void FlushClose(FileStream st) { if (st != null) { st.Flush(true); st.Close(); } } #endregion internal int NumberofBlocks() { return (int)((_datawrite.Length / _BLOCKSIZE)) + 1; } internal void FreeBlock(int i) { _freeList.Set(i, true); } } } ================================================ FILE: RaptorDB/Storage/StringHF.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace RaptorDB { // high frequency string value store public class StringHF //: IKeyStoreHF { internal class AllocationBlock { public string key; public byte keylen; public int datalength; public bool isCompressed; public bool isBinaryJSON; public bool deleteKey; public List Blocks = new List(); public int blocknumber; } //MGIndex _keys; StorageFileHF _datastore; object _lock = new object(); ushort _BlockSize = 2048; //private const int _KILOBYTE = 1024; ILog _log = LogManager.GetLogger(typeof(KeyStoreHF)); byte[] _blockheader = new byte[]{ 0,0,0,0, // 0 block # (used for validate block reads and rebuild) 0,0,0,0, // 4 next block # 0, // 8 flags bits 0:iscompressed 1:isbinary 2:deletekey 0,0,0,0, // 9 data length (compute alloc blocks needed) 0, // 13 key length 0, // 14 key type 0=guid 1=string }; private string _Path = ""; private string _S = Path.DirectorySeparatorChar.ToString(); //private bool _isDirty = false; //private string _dirtyFilename = "temp.$"; // mgindex special storage for strings ctor -> no idx file // use SaveData() GetData() public StringHF(string folder, string filename) { _Path = folder; Directory.CreateDirectory(_Path); if (_Path.EndsWith(_S) == false) _Path += _S; _datastore = new StorageFileHF(_Path + filename, Global.HighFrequencyKVDiskBlockSize); //_datastore.Initialize(); _BlockSize = _datastore.GetBlockSize(); } public void Shutdown() { _datastore.Shutdown(); //if (_keys != null) // _keys.Shutdown(); //if (File.Exists(_Path + _dirtyFilename)) // File.Delete(_Path + _dirtyFilename); } internal void FreeMemory() { //if (_keys != null) // _keys.FreeMemory(); } #region [ private methods ] private byte[] readblockdata(AllocationBlock ab) { byte[] data = new byte[ab.datalength]; long offset = 0; int len = ab.datalength; int dbsize = _BlockSize - _blockheader.Length - ab.keylen; ab.Blocks.ForEach(x => { byte[] b = _datastore.ReadBlock(x); int c = len; if (c > dbsize) c = dbsize; Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c); offset += c; len -= c; }); if (ab.isCompressed) data = MiniLZO.Decompress(data); return data; } private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab) { ab.Blocks = new List(); int firstblock = _datastore.GetFreeBlockNumber(); ab.Blocks.Add(firstblock); int blocknum = firstblock; byte[] header = CreateAllocHeader(ab, keybytes); int dblocksize = _BlockSize - header.Length; int offset = 0; // compute data block count int datablockcount = (data.Length / dblocksize) + 1; // save data blocks int counter = 0; int len = data.Length; while (datablockcount > 0) { datablockcount--; int next = 0; if (datablockcount > 0) next = _datastore.GetFreeBlockNumber(); Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4); // set block number Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4); // set next pointer _datastore.SeekBlock(blocknum); _datastore.WriteBlockBytes(header, 0, header.Length); int c = len; if (c > dblocksize) c = dblocksize; _datastore.WriteBlockBytes(data, offset, c); if (next > 0) { blocknum = next; ab.Blocks.Add(next); } offset += c; len -= c; counter++; } return firstblock; } private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes) { byte[] alloc = new byte[_blockheader.Length + keybytes.Length]; if (ab.isCompressed) alloc[8] = 1; if (ab.isBinaryJSON) alloc[8] += 2; if (ab.deleteKey) alloc[8] += 4; Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4); alloc[13] = ab.keylen; alloc[14] = 1; // string keys for now Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen); return alloc; } private AllocationBlock FillAllocationBlock(int blocknumber) { AllocationBlock ab = new AllocationBlock(); ab.blocknumber = blocknumber; ab.Blocks.Add(blocknumber); byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255); int blocknumexpected = 0; int next = ParseBlockHeader(ab, b, blocknumexpected); blocknumexpected++; while (next > 0) { ab.Blocks.Add(next); b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen); next = ParseBlockHeader(ab, b, blocknumexpected); blocknumexpected++; } return ab; } private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected) { int bnum = Helper.ToInt32(b, 0); if (bnum != blocknumberexpected) { _log.Error("Block numbers does not match, looking for : " + blocknumberexpected); //throw new Exception("Block numbers does not match, looking for : " + blocknumberexpected); return -1; } if (b[14] != 1) { _log.Error("Expecting string keys only, got : " + b[14]); //throw new Exception("Expecting string keys only, got : " + b[11]); return -1; } int next = Helper.ToInt32(b, 4); if (ab.keylen == 0) { byte flags = b[8]; if ((flags & 0x01) > 0) ab.isCompressed = true; if ((flags & 0x02) > 0) ab.isBinaryJSON = true; if ((flags & 0x04) > 0) ab.deleteKey = true; ab.datalength = Helper.ToInt32(b, 9); byte keylen = b[13]; ab.keylen = keylen; ab.key = Helper.GetString(b, _blockheader.Length, keylen); } return next; } #endregion internal void FreeBlocks(List list) { lock (_lock) _datastore.FreeBlocks(list); } // for .string files internal int SaveData(string key, byte[] data, out List blocks) { lock (_lock) { byte[] kb = Helper.GetBytes(key); AllocationBlock ab = new AllocationBlock(); ab.key = key; ab.keylen = (byte)kb.Length; ab.isCompressed = false; ab.isBinaryJSON = true; ab.datalength = data.Length; int firstblock = internalSave(kb, data, ab); blocks = ab.Blocks; return firstblock; } } // for .string files internal byte[] GetData(int blocknumber, out List usedblocks) { lock (_lock) { AllocationBlock ab = FillAllocationBlock(blocknumber); usedblocks = ab.Blocks; byte[] data = readblockdata(ab); return data; } } } } ================================================ FILE: RaptorDB/Views/Dynamic.cs ================================================ //Copyright (C) Microsoft Corporation. All rights reserved. using System.Collections.Generic; using System.Text; using System.Linq.Expressions; using System.Reflection; using System.Reflection.Emit; using System.Threading; namespace System.Linq.Dynamic { // FEATURE : cleanup unused code here #region [ Classes ] internal abstract class DynamicClass { public override string ToString() { PropertyInfo[] props = this.GetType().GetProperties(BindingFlags.Instance | BindingFlags.Public); StringBuilder sb = new StringBuilder(); sb.Append("{"); for (int i = 0; i < props.Length; i++) { if (i > 0) sb.Append(", "); sb.Append(props[i].Name); sb.Append("="); sb.Append(props[i].GetValue(this, null)); } sb.Append("}"); return sb.ToString(); } } internal class DynamicProperty { string name; Type type; public DynamicProperty(string name, Type type) { if (name == null) throw new ArgumentNullException("name"); if (type == null) throw new ArgumentNullException("type"); this.name = name; this.type = type; } public string Name { get { return name; } } public Type Type { get { return type; } } } internal static class DynamicExpression { public static Expression Parse(Type resultType, string expression, params object[] values) { ExpressionParser parser = new ExpressionParser(null, expression, values); return parser.Parse(resultType); } public static LambdaExpression ParseLambda(Type itType, Type resultType, string expression, params object[] values) { return ParseLambda(new ParameterExpression[] { Expression.Parameter(itType, "") }, resultType, expression, values); } public static LambdaExpression ParseLambda(ParameterExpression[] parameters, Type resultType, string expression, params object[] values) { ExpressionParser parser = new ExpressionParser(parameters, expression, values); return Expression.Lambda(parser.Parse(resultType), parameters); } //public static Expression> ParseLambda(string expression, params object[] values) //{ // return (Expression>)ParseLambda(typeof(T), typeof(S), expression, values); //} public static Type CreateClass(params DynamicProperty[] properties) { return ClassFactory.Instance.GetDynamicClass(properties); } public static Type CreateClass(IEnumerable properties) { return ClassFactory.Instance.GetDynamicClass(properties); } } //internal class DynamicOrdering //{ // public Expression Selector; // public bool Ascending; //} internal class Signature : IEquatable { public DynamicProperty[] properties; public int hashCode; public Signature(IEnumerable properties) { this.properties = properties.ToArray(); hashCode = 0; foreach (DynamicProperty p in properties) { hashCode ^= p.Name.GetHashCode() ^ p.Type.GetHashCode(); } } public override int GetHashCode() { return hashCode; } public override bool Equals(object obj) { return obj is Signature ? Equals((Signature)obj) : false; } public bool Equals(Signature other) { if (properties.Length != other.properties.Length) return false; for (int i = 0; i < properties.Length; i++) { if (properties[i].Name != other.properties[i].Name || properties[i].Type != other.properties[i].Type) return false; } return true; } } internal class ClassFactory { public static readonly ClassFactory Instance = new ClassFactory(); static ClassFactory() { } // Trigger lazy initialization of static fields ModuleBuilder module; Dictionary classes; int classCount; ReaderWriterLock rwLock; private ClassFactory() { AssemblyName name = new AssemblyName("DynamicClasses"); #if NETSTANDARD2_0 AssemblyBuilder assembly = AssemblyBuilder.DefineDynamicAssembly(name, AssemblyBuilderAccess.Run); #else AssemblyBuilder assembly = AppDomain.CurrentDomain.DefineDynamicAssembly(name, AssemblyBuilderAccess.Run); #endif #if ENABLE_LINQ_PARTIAL_TRUST new ReflectionPermission(PermissionState.Unrestricted).Assert(); #endif try { module = assembly.DefineDynamicModule("Module"); } finally { #if ENABLE_LINQ_PARTIAL_TRUST PermissionSet.RevertAssert(); #endif } classes = new Dictionary(); rwLock = new ReaderWriterLock(); } public Type GetDynamicClass(IEnumerable properties) { rwLock.AcquireReaderLock(Timeout.Infinite); try { Signature signature = new Signature(properties); Type type; if (!classes.TryGetValue(signature, out type)) { type = CreateDynamicClass(signature.properties); classes.Add(signature, type); } return type; } finally { rwLock.ReleaseReaderLock(); } } Type CreateDynamicClass(DynamicProperty[] properties) { LockCookie cookie = rwLock.UpgradeToWriterLock(Timeout.Infinite); try { string typeName = "DynamicClass" + (classCount + 1); #if ENABLE_LINQ_PARTIAL_TRUST new ReflectionPermission(PermissionState.Unrestricted).Assert(); #endif try { TypeBuilder tb = this.module.DefineType(typeName, TypeAttributes.Class | TypeAttributes.Public, typeof(DynamicClass)); FieldInfo[] fields = GenerateProperties(tb, properties); GenerateEquals(tb, fields); GenerateGetHashCode(tb, fields); #if NETSTANDARD2_0 Type result = tb.CreateTypeInfo(); #else Type result = tb.CreateType(); #endif classCount++; return result; } finally { #if ENABLE_LINQ_PARTIAL_TRUST PermissionSet.RevertAssert(); #endif } } finally { rwLock.DowngradeFromWriterLock(ref cookie); } } FieldInfo[] GenerateProperties(TypeBuilder tb, DynamicProperty[] properties) { FieldInfo[] fields = new FieldBuilder[properties.Length]; for (int i = 0; i < properties.Length; i++) { DynamicProperty dp = properties[i]; FieldBuilder fb = tb.DefineField("_" + dp.Name, dp.Type, FieldAttributes.Private); PropertyBuilder pb = tb.DefineProperty(dp.Name, PropertyAttributes.HasDefault, dp.Type, null); MethodBuilder mbGet = tb.DefineMethod("get_" + dp.Name, MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig, dp.Type, Type.EmptyTypes); ILGenerator genGet = mbGet.GetILGenerator(); genGet.Emit(OpCodes.Ldarg_0); genGet.Emit(OpCodes.Ldfld, fb); genGet.Emit(OpCodes.Ret); MethodBuilder mbSet = tb.DefineMethod("set_" + dp.Name, MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig, null, new Type[] { dp.Type }); ILGenerator genSet = mbSet.GetILGenerator(); genSet.Emit(OpCodes.Ldarg_0); genSet.Emit(OpCodes.Ldarg_1); genSet.Emit(OpCodes.Stfld, fb); genSet.Emit(OpCodes.Ret); pb.SetGetMethod(mbGet); pb.SetSetMethod(mbSet); fields[i] = fb; } return fields; } void GenerateEquals(TypeBuilder tb, FieldInfo[] fields) { MethodBuilder mb = tb.DefineMethod("Equals", MethodAttributes.Public | MethodAttributes.ReuseSlot | MethodAttributes.Virtual | MethodAttributes.HideBySig, typeof(bool), new Type[] { typeof(object) }); ILGenerator gen = mb.GetILGenerator(); LocalBuilder other = gen.DeclareLocal(tb); Label next = gen.DefineLabel(); gen.Emit(OpCodes.Ldarg_1); gen.Emit(OpCodes.Isinst, tb); gen.Emit(OpCodes.Stloc, other); gen.Emit(OpCodes.Ldloc, other); gen.Emit(OpCodes.Brtrue_S, next); gen.Emit(OpCodes.Ldc_I4_0); gen.Emit(OpCodes.Ret); gen.MarkLabel(next); foreach (FieldInfo field in fields) { Type ft = field.FieldType; Type ct = typeof(EqualityComparer<>).MakeGenericType(ft); next = gen.DefineLabel(); gen.EmitCall(OpCodes.Call, ct.GetMethod("get_Default"), null); gen.Emit(OpCodes.Ldarg_0); gen.Emit(OpCodes.Ldfld, field); gen.Emit(OpCodes.Ldloc, other); gen.Emit(OpCodes.Ldfld, field); gen.EmitCall(OpCodes.Callvirt, ct.GetMethod("Equals", new Type[] { ft, ft }), null); gen.Emit(OpCodes.Brtrue_S, next); gen.Emit(OpCodes.Ldc_I4_0); gen.Emit(OpCodes.Ret); gen.MarkLabel(next); } gen.Emit(OpCodes.Ldc_I4_1); gen.Emit(OpCodes.Ret); } void GenerateGetHashCode(TypeBuilder tb, FieldInfo[] fields) { MethodBuilder mb = tb.DefineMethod("GetHashCode", MethodAttributes.Public | MethodAttributes.ReuseSlot | MethodAttributes.Virtual | MethodAttributes.HideBySig, typeof(int), Type.EmptyTypes); ILGenerator gen = mb.GetILGenerator(); gen.Emit(OpCodes.Ldc_I4_0); foreach (FieldInfo field in fields) { Type ft = field.FieldType; Type ct = typeof(EqualityComparer<>).MakeGenericType(ft); gen.EmitCall(OpCodes.Call, ct.GetMethod("get_Default"), null); gen.Emit(OpCodes.Ldarg_0); gen.Emit(OpCodes.Ldfld, field); gen.EmitCall(OpCodes.Callvirt, ct.GetMethod("GetHashCode", new Type[] { ft }), null); gen.Emit(OpCodes.Xor); } gen.Emit(OpCodes.Ret); } } internal sealed class ParseException : Exception { int position; public ParseException(string message, int position) : base(message) { this.position = position; } public int Position { get { return position; } } public override string ToString() { return string.Format(Res.ParseExceptionFormat, Message, position); } } #endregion internal class ExpressionParser { #region [ internal ] struct Token { public TokenId id; public string text; public int pos; } enum TokenId { Unknown, End, Identifier, StringLiteral, IntegerLiteral, RealLiteral, Exclamation, Percent, Amphersand, OpenParen, CloseParen, Asterisk, Plus, Comma, Minus, Dot, Slash, Colon, LessThan, Equal, GreaterThan, Question, OpenBracket, CloseBracket, Bar, ExclamationEqual, DoubleAmphersand, LessThanEqual, LessGreater, DoubleEqual, GreaterThanEqual, DoubleBar } interface ILogicalSignatures { void F(bool x, bool y); void F(bool? x, bool? y); } interface IArithmeticSignatures { void F(int x, int y); void F(uint x, uint y); void F(long x, long y); void F(ulong x, ulong y); void F(float x, float y); void F(double x, double y); void F(decimal x, decimal y); void F(int? x, int? y); void F(uint? x, uint? y); void F(long? x, long? y); void F(ulong? x, ulong? y); void F(float? x, float? y); void F(double? x, double? y); void F(decimal? x, decimal? y); } interface IRelationalSignatures : IArithmeticSignatures { void F(string x, string y); void F(char x, char y); void F(DateTime x, DateTime y); void F(TimeSpan x, TimeSpan y); void F(char? x, char? y); void F(DateTime? x, DateTime? y); void F(TimeSpan? x, TimeSpan? y); } interface IEqualitySignatures : IRelationalSignatures { void F(bool x, bool y); void F(bool? x, bool? y); } interface IAddSignatures : IArithmeticSignatures { void F(DateTime x, TimeSpan y); void F(TimeSpan x, TimeSpan y); void F(DateTime? x, TimeSpan? y); void F(TimeSpan? x, TimeSpan? y); } interface ISubtractSignatures : IAddSignatures { void F(DateTime x, DateTime y); void F(DateTime? x, DateTime? y); } interface INegationSignatures { void F(int x); void F(long x); void F(float x); void F(double x); void F(decimal x); void F(int? x); void F(long? x); void F(float? x); void F(double? x); void F(decimal? x); } interface INotSignatures { void F(bool x); void F(bool? x); } interface IEnumerableSignatures { void Where(bool predicate); void Any(); void Any(bool predicate); void All(bool predicate); void Count(); void Count(bool predicate); void Min(object selector); void Max(object selector); void Sum(int selector); void Sum(int? selector); void Sum(long selector); void Sum(long? selector); void Sum(float selector); void Sum(float? selector); void Sum(double selector); void Sum(double? selector); void Sum(decimal selector); void Sum(decimal? selector); void Average(int selector); void Average(int? selector); void Average(long selector); void Average(long? selector); void Average(float selector); void Average(float? selector); void Average(double selector); void Average(double? selector); void Average(decimal selector); void Average(decimal? selector); } static readonly Type[] predefinedTypes = { typeof(Object), typeof(Boolean), typeof(Char), typeof(String), typeof(SByte), typeof(Byte), typeof(Int16), typeof(UInt16), typeof(Int32), typeof(UInt32), typeof(Int64), typeof(UInt64), typeof(Single), typeof(Double), typeof(Decimal), typeof(DateTime), typeof(TimeSpan), typeof(Guid), typeof(Math), typeof(Convert) }; #endregion #region [ gunk ] //// *, /, %, mod operators //Expression ParseMultiplicative() //{ // Expression left = ParseUnary(); // //while (token.id == TokenId.Asterisk || token.id == TokenId.Slash || // // token.id == TokenId.Percent || TokenIdentifierIs("mod")) // //{ // // Token op = token; // // NextToken(); // // Expression right = ParseUnary(); // // CheckAndPromoteOperands(typeof(IArithmeticSignatures), op.text, ref left, ref right, op.pos); // // switch (op.id) // // { // // case TokenId.Asterisk: // // left = Expression.Multiply(left, right); // // break; // // case TokenId.Slash: // // left = Expression.Divide(left, right); // // break; // // case TokenId.Percent: // // case TokenId.Identifier: // // left = Expression.Modulo(left, right); // // break; // // } // //} // return left; //} //Expression GenerateAdd(Expression left, Expression right) //{ // if (left.Type == typeof(string) && right.Type == typeof(string)) // { // return GenerateStaticMethodCall("Concat", left, right); // } // return Expression.Add(left, right); //} //Expression GenerateSubtract(Expression left, Expression right) //{ // return Expression.Subtract(left, right); //} //Expression GenerateStringConcat(Expression left, Expression right) //{ // return Expression.Call( // null, // typeof(string).GetMethod("Concat", new[] { typeof(object), typeof(object) }), // new[] { left, right }); //} #pragma warning disable 0219 //public IEnumerable ParseOrdering() //{ // List orderings = new List(); // while (true) // { // Expression expr = ParseExpression(); // bool ascending = true; // if (TokenIdentifierIs("asc") || TokenIdentifierIs("ascending")) // { // NextToken(); // } // else if (TokenIdentifierIs("desc") || TokenIdentifierIs("descending")) // { // NextToken(); // ascending = false; // } // orderings.Add(new DynamicOrdering { Selector = expr, Ascending = ascending }); // if (token.id != TokenId.Comma) break; // NextToken(); // } // ValidateToken(TokenId.End, Res.SyntaxError); // return orderings; //} #pragma warning restore 0219 //Expression ParseIt() //{ // if (it == null) // throw ParseError(Res.NoItInScope); // NextToken(); // return it; //} //Expression ParseIif() //{ // int errorPos = token.pos; // NextToken(); // Expression[] args = ParseArgumentList(); // if (args.Length != 3) // throw ParseError(errorPos, Res.IifRequiresThreeArgs); // return GenerateConditional(args[0], args[1], args[2], errorPos); //} //Expression GenerateConditional(Expression test, Expression expr1, Expression expr2, int errorPos) //{ // if (test.Type != typeof(bool)) // throw ParseError(errorPos, Res.FirstExprMustBeBool); // if (expr1.Type != expr2.Type) // { // Expression expr1as2 = expr2 != nullLiteral ? PromoteExpression(expr1, expr2.Type, true) : null; // Expression expr2as1 = expr1 != nullLiteral ? PromoteExpression(expr2, expr1.Type, true) : null; // if (expr1as2 != null && expr2as1 == null) // { // expr1 = expr1as2; // } // else if (expr2as1 != null && expr1as2 == null) // { // expr2 = expr2as1; // } // else // { // string type1 = expr1 != nullLiteral ? expr1.Type.Name : "null"; // string type2 = expr2 != nullLiteral ? expr2.Type.Name : "null"; // if (expr1as2 != null && expr2as1 != null) // throw ParseError(errorPos, Res.BothTypesConvertToOther, type1, type2); // throw ParseError(errorPos, Res.NeitherTypeConvertsToOther, type1, type2); // } // } // return Expression.Condition(test, expr1, expr2); //} //Expression ParseNew() //{ // NextToken(); // ValidateToken(TokenId.OpenParen, Res.OpenParenExpected); // NextToken(); // List properties = new List(); // List expressions = new List(); // while (true) // { // int exprPos = token.pos; // Expression expr = ParseExpression(); // string propName; // if (TokenIdentifierIs("as")) // { // NextToken(); // propName = GetIdentifier(); // NextToken(); // } // else // { // MemberExpression me = expr as MemberExpression; // if (me == null) throw ParseError(exprPos, Res.MissingAsClause); // propName = me.Member.Name; // } // expressions.Add(expr); // properties.Add(new DynamicProperty(propName, expr.Type)); // if (token.id != TokenId.Comma) break; // NextToken(); // } // ValidateToken(TokenId.CloseParen, Res.CloseParenOrCommaExpected); // NextToken(); // Type type = DynamicExpression.CreateClass(properties); // MemberBinding[] bindings = new MemberBinding[properties.Count]; // for (int i = 0; i < bindings.Length; i++) // bindings[i] = Expression.Bind(type.GetProperty(properties[i].Name), expressions[i]); // return Expression.MemberInit(Expression.New(type), bindings); //} //Expression ParseLambdaInvocation(LambdaExpression lambda) //{ // int errorPos = token.pos; // NextToken(); // Expression[] args = ParseArgumentList(); // MethodBase method; // if (FindMethod(lambda.Type, "Invoke", false, args, out method) != 1) // throw ParseError(errorPos, Res.ArgsIncompatibleWithLambda); // return Expression.Invoke(lambda, args); //} //Expression GenerateConversion(Expression expr, Type type, int errorPos) //{ // Type exprType = expr.Type; // if (exprType == type) return expr; // if (exprType.IsValueType && type.IsValueType) // { // if ((IsNullableType(exprType) || IsNullableType(type)) && // GetNonNullableType(exprType) == GetNonNullableType(type)) // return Expression.Convert(expr, type); // if ((IsNumericType(exprType) || IsEnumType(exprType)) && // (IsNumericType(type)) || IsEnumType(type)) // return Expression.ConvertChecked(expr, type); // } // if (exprType.IsAssignableFrom(type) || type.IsAssignableFrom(exprType) || // exprType.IsInterface || type.IsInterface) // return Expression.Convert(expr, type); // throw ParseError(errorPos, Res.CannotConvertValue, // GetTypeName(exprType), GetTypeName(type)); //} //static Type FindGenericType(Type generic, Type type) //{ // while (type != null && type != typeof(object)) // { // if (type.IsGenericType && type.GetGenericTypeDefinition() == generic) return type; // if (generic.IsInterface) // { // foreach (Type intfType in type.GetInterfaces()) // { // Type found = FindGenericType(generic, intfType); // if (found != null) return found; // } // } // type = type.BaseType; // } // return null; //} //Expression ParseAggregate(Expression instance, Type elementType, string methodName, int errorPos) //{ // ParameterExpression outerIt = it; // ParameterExpression innerIt = Expression.Parameter(elementType, ""); // it = innerIt; // Expression[] args = ParseArgumentList(); // it = outerIt; // MethodBase signature; // if (FindMethod(typeof(IEnumerableSignatures), methodName, false, args, out signature) != 1) // throw ParseError(errorPos, Res.NoApplicableAggregate, methodName); // Type[] typeArgs; // if (signature.Name == "Min" || signature.Name == "Max") // { // typeArgs = new Type[] { elementType, args[0].Type }; // } // else // { // typeArgs = new Type[] { elementType }; // } // if (args.Length == 0) // { // args = new Expression[] { instance }; // } // else // { // args = new Expression[] { instance, Expression.Lambda(args[0], innerIt) }; // } // return Expression.Call(typeof(Enumerable), signature.Name, typeArgs, args); //} Expression[] ParseArgumentList() { ValidateToken(TokenId.OpenParen, Res.OpenParenExpected); NextToken(); Expression[] args = token.id != TokenId.CloseParen ? ParseArguments() : new Expression[0]; ValidateToken(TokenId.CloseParen, Res.CloseParenOrCommaExpected); NextToken(); return args; } Expression[] ParseArguments() { List argList = new List(); while (true) { argList.Add(ParseExpression()); if (token.id != TokenId.Comma) break; NextToken(); } return argList.ToArray(); } //Expression ParseElementAccess(Expression expr) //{ // int errorPos = token.pos; // ValidateToken(TokenId.OpenBracket, Res.OpenParenExpected); // NextToken(); // Expression[] args = ParseArguments(); // ValidateToken(TokenId.CloseBracket, Res.CloseBracketOrCommaExpected); // NextToken(); // if (expr.Type.IsArray) // { // if (expr.Type.GetArrayRank() != 1 || args.Length != 1) // throw ParseError(errorPos, Res.CannotIndexMultiDimArray); // Expression index = PromoteExpression(args[0], typeof(int), true); // if (index == null) // throw ParseError(errorPos, Res.InvalidIndex); // return Expression.ArrayIndex(expr, index); // } // else // { // MethodBase mb; // switch (FindIndexer(expr.Type, args, out mb)) // { // case 0: // throw ParseError(errorPos, Res.NoApplicableIndexer, // GetTypeName(expr.Type)); // case 1: // return Expression.Call(expr, (MethodInfo)mb, args); // default: // throw ParseError(errorPos, Res.AmbiguousIndexerInvocation, // GetTypeName(expr.Type)); // } // } //} //static bool IsPredefinedType(Type type) //{ // foreach (Type t in predefinedTypes) if (t == type) return true; // return false; //} //static bool IsNumericType(Type type) //{ // return GetNumericTypeKind(type) != 0; //} //static bool IsEnumType(Type type) //{ // return GetNonNullableType(type).IsEnum; //} //static readonly string keywordIt = "it"; //static readonly string keywordIif = "iif"; //static readonly string keywordNew = "new"; #endregion static readonly Expression trueLiteral = Expression.Constant(true); static readonly Expression falseLiteral = Expression.Constant(false); static readonly Expression nullLiteral = Expression.Constant(null); static Dictionary keywords; Dictionary symbols; IDictionary externals; Dictionary literals; ParameterExpression it; string text; int textPos; int textLen; char ch; Token token; public ExpressionParser(ParameterExpression[] parameters, string expression, object[] values) { if (expression == null) throw new ArgumentNullException("expression"); if (keywords == null) keywords = CreateKeywords(); symbols = new Dictionary(StringComparer.OrdinalIgnoreCase); literals = new Dictionary(); if (parameters != null) ProcessParameters(parameters); if (values != null) ProcessValues(values); text = expression; textLen = text.Length; SetTextPos(0); NextToken(); } void ProcessParameters(ParameterExpression[] parameters) { foreach (ParameterExpression pe in parameters) if (!String.IsNullOrEmpty(pe.Name)) AddSymbol(pe.Name, pe); if (parameters.Length == 1 && String.IsNullOrEmpty(parameters[0].Name)) it = parameters[0]; } void ProcessValues(object[] values) { for (int i = 0; i < values.Length; i++) { object value = values[i]; if (i == values.Length - 1 && value is IDictionary) { externals = (IDictionary)value; } else { AddSymbol("@" + i.ToString(System.Globalization.CultureInfo.InvariantCulture), value); } } } void AddSymbol(string name, object value) { if (symbols.ContainsKey(name)) throw ParseError(Res.DuplicateIdentifier, name); symbols.Add(name, value); } public Expression Parse(Type resultType) { int exprPos = token.pos; Expression expr = ParseExpression(); if (resultType != null) if ((expr = PromoteExpression(expr, resultType, true)) == null) throw ParseError(exprPos, Res.ExpressionTypeMismatch, GetTypeName(resultType)); ValidateToken(TokenId.End, Res.SyntaxError); return expr; } // ?: operator Expression ParseExpression() { int errorPos = token.pos; Expression expr = ParseLogicalOr(); //if (token.id == TokenId.Question) //{ // NextToken(); // Expression expr1 = ParseExpression(); // ValidateToken(TokenId.Colon, Res.ColonExpected); // NextToken(); // Expression expr2 = ParseExpression(); // expr = GenerateConditional(expr, expr1, expr2, errorPos); //} return expr; } // ||, or operator Expression ParseLogicalOr() { Expression left = ParseLogicalAnd(); while (token.id == TokenId.DoubleBar || TokenIdentifierIs("or")) { Token op = token; NextToken(); Expression right = ParseLogicalAnd(); CheckAndPromoteOperands(typeof(ILogicalSignatures), op.text, ref left, ref right, op.pos); left = Expression.OrElse(left, right); } return left; } // &&, and operator Expression ParseLogicalAnd() { Expression left = ParseComparison(); while (token.id == TokenId.DoubleAmphersand || TokenIdentifierIs("and")) { Token op = token; NextToken(); Expression right = ParseComparison(); CheckAndPromoteOperands(typeof(ILogicalSignatures), op.text, ref left, ref right, op.pos); left = Expression.AndAlso(left, right); } return left; } // =, ==, !=, <>, >, >=, <, <= operators Expression ParseComparison() { Expression left = ParseAdditive(); while (token.id == TokenId.Equal || token.id == TokenId.DoubleEqual || token.id == TokenId.ExclamationEqual || token.id == TokenId.LessGreater || token.id == TokenId.GreaterThan || token.id == TokenId.GreaterThanEqual || token.id == TokenId.LessThan || token.id == TokenId.LessThanEqual) { Token op = token; NextToken(); Expression right = ParseAdditive(); bool isEquality = op.id == TokenId.Equal || op.id == TokenId.DoubleEqual || op.id == TokenId.ExclamationEqual || op.id == TokenId.LessGreater; if (isEquality && !left.Type.IsValueType && !right.Type.IsValueType) { if (left.Type != right.Type) { if (left.Type.IsAssignableFrom(right.Type)) { right = Expression.Convert(right, left.Type); } else if (right.Type.IsAssignableFrom(left.Type)) { left = Expression.Convert(left, right.Type); } else { throw IncompatibleOperandsError(op.text, left, right, op.pos); } } } //else if (IsEnumType(left.Type) || IsEnumType(right.Type)) //{ // if (left.Type != right.Type) // { // Expression e; // if ((e = PromoteExpression(right, left.Type, true)) != null) // { // right = e; // } // else if ((e = PromoteExpression(left, right.Type, true)) != null) // { // left = e; // } // else // { // throw IncompatibleOperandsError(op.text, left, right, op.pos); // } // } //} else { if (left.Type == typeof(Guid)) right = Expression.Constant(Guid.Parse(right.ToString().Replace("\"", "").Replace("'", ""))); else if (left.Type == typeof(DateTime)) right = Expression.Constant(DateTime.Parse(right.ToString().Replace("\"", "").Replace("'", ""))); else CheckAndPromoteOperands(isEquality ? typeof(IEqualitySignatures) : typeof(IRelationalSignatures), op.text, ref left, ref right, op.pos); } switch (op.id) { case TokenId.Equal: case TokenId.DoubleEqual: left = GenerateEqual(left, right); break; case TokenId.ExclamationEqual: case TokenId.LessGreater: left = GenerateNotEqual(left, right); break; case TokenId.GreaterThan: left = GenerateGreaterThan(left, right); break; case TokenId.GreaterThanEqual: left = GenerateGreaterThanEqual(left, right); break; case TokenId.LessThan: left = GenerateLessThan(left, right); break; case TokenId.LessThanEqual: left = GenerateLessThanEqual(left, right); break; } } return left; } // +, -, & operators Expression ParseAdditive() { Expression left = ParseUnary();// ParseMultiplicative(); //while (token.id == TokenId.Plus || token.id == TokenId.Minus || // token.id == TokenId.Amphersand) //{ // Token op = token; // NextToken(); // Expression right = ParseMultiplicative(); // switch (op.id) // { // case TokenId.Plus: // if (left.Type == typeof(string) || right.Type == typeof(string)) // goto case TokenId.Amphersand; // CheckAndPromoteOperands(typeof(IAddSignatures), op.text, ref left, ref right, op.pos); // left = GenerateAdd(left, right); // break; // case TokenId.Minus: // CheckAndPromoteOperands(typeof(ISubtractSignatures), op.text, ref left, ref right, op.pos); // left = GenerateSubtract(left, right); // break; // case TokenId.Amphersand: // left = GenerateStringConcat(left, right); // break; // } //} return left; } // -, !, not unary operators Expression ParseUnary() { if (token.id == TokenId.Minus || token.id == TokenId.Exclamation || TokenIdentifierIs("not")) { Token op = token; NextToken(); if (op.id == TokenId.Minus && (token.id == TokenId.IntegerLiteral || token.id == TokenId.RealLiteral)) { token.text = "-" + token.text; token.pos = op.pos; return ParsePrimary(); } Expression expr = ParseUnary(); if (op.id == TokenId.Minus) { CheckAndPromoteOperand(typeof(INegationSignatures), op.text, ref expr, op.pos); expr = Expression.Negate(expr); } else { CheckAndPromoteOperand(typeof(INotSignatures), op.text, ref expr, op.pos); expr = Expression.Not(expr); } return expr; } return ParsePrimary(); } Expression ParsePrimary() { Expression expr = ParsePrimaryStart(); while (true) { if (token.id == TokenId.Dot) { NextToken(); expr = ParseMemberAccess(null, expr); } //else if (token.id == TokenId.OpenBracket) //{ // expr = ParseElementAccess(expr); //} else { break; } } return expr; } Expression ParsePrimaryStart() { switch (token.id) { case TokenId.Identifier: return ParseIdentifier(); case TokenId.StringLiteral: return ParseStringLiteral(); case TokenId.IntegerLiteral: return ParseIntegerLiteral(); case TokenId.RealLiteral: return ParseRealLiteral(); case TokenId.OpenParen: return ParseParenExpression(); default: throw ParseError(Res.ExpressionExpected); } } Expression ParseStringLiteral() { ValidateToken(TokenId.StringLiteral); char quote = token.text[0]; string s = token.text.Substring(1, token.text.Length - 2); int start = 0; while (true) { int i = s.IndexOf(quote, start); if (i < 0) break; s = s.Remove(i, 1); start = i + 1; } if (quote == '\'') { if (s.Length != 1) throw ParseError(Res.InvalidCharacterLiteral); NextToken(); return CreateLiteral(s[0], s); } NextToken(); return CreateLiteral(s, s); } Expression ParseIntegerLiteral() { ValidateToken(TokenId.IntegerLiteral); string text = token.text; if (text[0] != '-') { ulong value; if (!UInt64.TryParse(text, out value)) throw ParseError(Res.InvalidIntegerLiteral, text); NextToken(); if (value <= (ulong)Int32.MaxValue) return CreateLiteral((int)value, text); if (value <= (ulong)UInt32.MaxValue) return CreateLiteral((uint)value, text); if (value <= (ulong)Int64.MaxValue) return CreateLiteral((long)value, text); return CreateLiteral(value, text); } else { long value; if (!Int64.TryParse(text, out value)) throw ParseError(Res.InvalidIntegerLiteral, text); NextToken(); if (value >= Int32.MinValue && value <= Int32.MaxValue) return CreateLiteral((int)value, text); return CreateLiteral(value, text); } } Expression ParseRealLiteral() { ValidateToken(TokenId.RealLiteral); string text = token.text; object value = null; char last = text[text.Length - 1]; if (last == 'F' || last == 'f') { float f; if (Single.TryParse(text.Substring(0, text.Length - 1), out f)) value = f; } else { double d; if (Double.TryParse(text, out d)) value = d; } if (value == null) throw ParseError(Res.InvalidRealLiteral, text); NextToken(); return CreateLiteral(value, text); } Expression CreateLiteral(object value, string text) { ConstantExpression expr = Expression.Constant(value); literals.Add(expr, text); return expr; } Expression ParseParenExpression() { ValidateToken(TokenId.OpenParen, Res.OpenParenExpected); NextToken(); Expression e = ParseExpression(); ValidateToken(TokenId.CloseParen, Res.CloseParenOrOperatorExpected); NextToken(); return e; } Expression ParseIdentifier() { ValidateToken(TokenId.Identifier); object value; if (keywords.TryGetValue(token.text, out value)) { if (value is Type) return ParseTypeAccess((Type)value); //if (value == (object)keywordIt) return ParseIt(); //if (value == (object)keywordIif) return ParseIif(); //if (value == (object)keywordNew) return ParseNew(); NextToken(); return (Expression)value; } if (symbols.TryGetValue(token.text, out value) || externals != null && externals.TryGetValue(token.text, out value)) { Expression expr = value as Expression; if (expr == null) { expr = Expression.Constant(value); } //else //{ // LambdaExpression lambda = expr as LambdaExpression; // if (lambda != null) return ParseLambdaInvocation(lambda); //} NextToken(); return expr; } if (it != null) return ParseMemberAccess(null, it); throw ParseError(Res.UnknownIdentifier, token.text); } Expression ParseTypeAccess(Type type) { int errorPos = token.pos; NextToken(); if (token.id == TokenId.Question) { if (!type.IsValueType || IsNullableType(type)) throw ParseError(errorPos, Res.TypeHasNoNullableForm, GetTypeName(type)); type = typeof(Nullable<>).MakeGenericType(type); NextToken(); } //if (token.id == TokenId.OpenParen) //{ // Expression[] args = ParseArgumentList(); // MethodBase method; // switch (FindBestMethod(type.GetConstructors(), args, out method)) // { // case 0: // if (args.Length == 1) // return GenerateConversion(args[0], type, errorPos); // throw ParseError(errorPos, Res.NoMatchingConstructor, GetTypeName(type)); // case 1: // return Expression.New((ConstructorInfo)method, args); // default: // throw ParseError(errorPos, Res.AmbiguousConstructorInvocation, GetTypeName(type)); // } //} ValidateToken(TokenId.Dot, Res.DotOrOpenParenExpected); NextToken(); return ParseMemberAccess(type, null); } internal static Dictionary _betweens = new Dictionary(); Expression CallBetween(Expression instance, Type checktype, Type datatype, Expression[] args) { MethodInfo method = null; _betweens.TryGetValue(datatype, out method); if (method == null) foreach (var m in typeof(RDBExtension).GetMethods()) { if (m.Name.ToLower() == "between") { var pi = m.GetParameters(); if (pi[1].ParameterType == checktype) { method = m; _betweens.Add(datatype, method); break; } else if (pi[1].ParameterType.IsGenericParameter) { method = m.MakeGenericMethod(args[0].Type); _betweens.Add(datatype, method); break; } } } var pname = instance.ToString(); return Expression.Call(method, new Expression[] { ConstantExpression.Variable(datatype, pname), args[0], args[1] }); } internal static Dictionary _ins = new Dictionary(); Expression CallIn(Expression instance, Type checktype, Type datatype, Expression[] args) { MethodInfo method = null; _ins.TryGetValue(datatype, out method); if (method == null) foreach (var m in typeof(RDBExtension).GetMethods()) { if (m.Name.ToLower() == "in") { var pi = m.GetParameters(); if (pi[0].ParameterType.IsGenericParameter) { method = m.MakeGenericMethod(datatype); _ins.Add(datatype, method); break; } } } var pname = instance.ToString(); // calling on byte column status.in(1,3) 1,3 = int32 not byte if(datatype != args[0].Type) { for (int i = 0; i < args.Length; i++) { args[i] = ConstantExpression.Constant( Convert.ChangeType( (args[i] as ConstantExpression).Value, datatype)); } } return Expression.Call(method, new Expression[] { ConstantExpression.Variable(datatype, pname), Expression.NewArrayInit(datatype, args) }); } Expression ParseMemberAccess(Type type, Expression instance) { if (instance != null) type = instance.Type; int errorPos = token.pos; string id = GetIdentifier(); NextToken(); #region commented //if (token.id == TokenId.OpenParen) //{ // //if (instance != null && type != typeof(string)) // //{ // // Type enumerableType = FindGenericType(typeof(IEnumerable<>), type); // // if (enumerableType != null) // // { // // Type elementType = enumerableType.GetGenericArguments()[0]; // // return ParseAggregate(instance, elementType, id, errorPos); // // } // //} // Expression[] args = ParseArgumentList(); // MethodBase mb; // switch (FindMethod(type, id, instance == null, args, out mb)) // { // case 0: // throw ParseError(errorPos, Res.NoApplicableMethod, // id, GetTypeName(type)); // case 1: // MethodInfo method = (MethodInfo)mb; // if (!IsPredefinedType(method.DeclaringType)) // throw ParseError(errorPos, Res.MethodsAreInaccessible, GetTypeName(method.DeclaringType)); // if (method.ReturnType == typeof(void)) // throw ParseError(errorPos, Res.MethodIsVoid, // id, GetTypeName(method.DeclaringType)); // return Expression.Call(instance, (MethodInfo)method, args); // default: // throw ParseError(errorPos, Res.AmbiguousMethodInvocation, // id, GetTypeName(type)); // } //} //else #endregion { MemberInfo member = FindPropertyOrField(type, id, instance == null); if (member == null) { if (id.ToLower() == "between") { Expression[] args = ParseArgumentList(); if (type == typeof(DateTime)) return CallBetween(instance, typeof(string), typeof(DateTime), args); else return CallBetween(instance, type, type, args); } else if(id.ToLower() == "in") // in(1,3,5,7) { var args = ParseArgumentList(); int i = args.Length; return CallIn(instance, type, type, args); } else throw ParseError(errorPos, Res.UnknownPropertyOrField, id, GetTypeName(type)); } return member is PropertyInfo ? Expression.Property(instance, (PropertyInfo)member) : Expression.Field(instance, (FieldInfo)member); } } static bool IsNullableType(Type type) { return type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>); } static Type GetNonNullableType(Type type) { return IsNullableType(type) ? type.GetGenericArguments()[0] : type; } static string GetTypeName(Type type) { Type baseType = GetNonNullableType(type); string s = baseType.Name; if (type != baseType) s += '?'; return s; } static bool IsSignedIntegralType(Type type) { return GetNumericTypeKind(type) == 2; } static bool IsUnsignedIntegralType(Type type) { return GetNumericTypeKind(type) == 3; } static int GetNumericTypeKind(Type type) { type = GetNonNullableType(type); if (type.IsEnum) return 0; switch (Type.GetTypeCode(type)) { case TypeCode.Char: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return 1; case TypeCode.SByte: case TypeCode.Int16: case TypeCode.Int32: case TypeCode.Int64: return 2; case TypeCode.Byte: case TypeCode.UInt16: case TypeCode.UInt32: case TypeCode.UInt64: return 3; default: return 0; } } void CheckAndPromoteOperand(Type signatures, string opName, ref Expression expr, int errorPos) { Expression[] args = new Expression[] { expr }; MethodBase method; if (FindMethod(signatures, "F", false, args, out method) != 1) throw ParseError(errorPos, Res.IncompatibleOperand, opName, GetTypeName(args[0].Type)); expr = args[0]; } void CheckAndPromoteOperands(Type signatures, string opName, ref Expression left, ref Expression right, int errorPos) { Expression[] args = new Expression[] { left, right }; MethodBase method; if (FindMethod(signatures, "F", false, args, out method) != 1) throw IncompatibleOperandsError(opName, left, right, errorPos); left = args[0]; right = args[1]; } Exception IncompatibleOperandsError(string opName, Expression left, Expression right, int pos) { return ParseError(pos, Res.IncompatibleOperands, opName, GetTypeName(left.Type), GetTypeName(right.Type)); } MemberInfo FindPropertyOrField(Type type, string memberName, bool staticAccess) { BindingFlags flags = BindingFlags.Public | BindingFlags.DeclaredOnly | (staticAccess ? BindingFlags.Static : BindingFlags.Instance); foreach (Type t in SelfAndBaseTypes(type)) { MemberInfo[] members = t.FindMembers(MemberTypes.Property | MemberTypes.Field, flags, Type.FilterNameIgnoreCase, memberName); if (members.Length != 0) return members[0]; } return null; } int FindMethod(Type type, string methodName, bool staticAccess, Expression[] args, out MethodBase method) { BindingFlags flags = BindingFlags.Public | BindingFlags.DeclaredOnly | (staticAccess ? BindingFlags.Static : BindingFlags.Instance); foreach (Type t in SelfAndBaseTypes(type)) { MemberInfo[] members = t.FindMembers(MemberTypes.Method, flags, Type.FilterNameIgnoreCase, methodName); int count = FindBestMethod(members.Cast(), args, out method); if (count != 0) return count; } method = null; return 0; } int FindIndexer(Type type, Expression[] args, out MethodBase method) { foreach (Type t in SelfAndBaseTypes(type)) { MemberInfo[] members = t.GetDefaultMembers(); if (members.Length != 0) { IEnumerable methods = members. OfType(). Select(p => (MethodBase)p.GetGetMethod()). Where(m => m != null); int count = FindBestMethod(methods, args, out method); if (count != 0) return count; } } method = null; return 0; } static IEnumerable SelfAndBaseTypes(Type type) { if (type.IsInterface) { List types = new List(); AddInterface(types, type); return types; } return SelfAndBaseClasses(type); } static IEnumerable SelfAndBaseClasses(Type type) { while (type != null) { yield return type; type = type.BaseType; } } static void AddInterface(List types, Type type) { if (!types.Contains(type)) { types.Add(type); foreach (Type t in type.GetInterfaces()) AddInterface(types, t); } } class MethodData { public MethodBase MethodBase; public ParameterInfo[] Parameters; public Expression[] Args; } int FindBestMethod(IEnumerable methods, Expression[] args, out MethodBase method) { MethodData[] applicable = methods. Select(m => new MethodData { MethodBase = m, Parameters = m.GetParameters() }). Where(m => IsApplicable(m, args)). ToArray(); if (applicable.Length > 1) { applicable = applicable. Where(m => applicable.All(n => m == n || IsBetterThan(args, m, n))). ToArray(); } if (applicable.Length == 1) { MethodData md = applicable[0]; for (int i = 0; i < args.Length; i++) args[i] = md.Args[i]; method = md.MethodBase; } else { method = null; } return applicable.Length; } bool IsApplicable(MethodData method, Expression[] args) { if (method.Parameters.Length != args.Length) return false; Expression[] promotedArgs = new Expression[args.Length]; for (int i = 0; i < args.Length; i++) { ParameterInfo pi = method.Parameters[i]; if (pi.IsOut) return false; Expression promoted = PromoteExpression(args[i], pi.ParameterType, false); if (promoted == null) return false; promotedArgs[i] = promoted; } method.Args = promotedArgs; return true; } Expression PromoteExpression(Expression expr, Type type, bool exact) { if (expr.Type == type) return expr; if (expr is ConstantExpression) { ConstantExpression ce = (ConstantExpression)expr; if (ce == nullLiteral) { if (!type.IsValueType || IsNullableType(type)) return Expression.Constant(null, type); } else { string text; if (literals.TryGetValue(ce, out text)) { Type target = GetNonNullableType(type); Object value = null; switch (Type.GetTypeCode(ce.Type)) { case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: value = ParseNumber(text, target); break; case TypeCode.Double: if (target == typeof(decimal)) value = ParseNumber(text, target); break; case TypeCode.String: value = ParseEnum(text, target); break; } if (value != null) return Expression.Constant(value, type); } } } if (IsCompatibleWith(expr.Type, type)) { if (type.IsValueType || exact) return Expression.Convert(expr, type); return expr; } return null; } static object ParseNumber(string text, Type type) { switch (Type.GetTypeCode(GetNonNullableType(type))) { case TypeCode.SByte: sbyte sb; if (sbyte.TryParse(text, out sb)) return sb; break; case TypeCode.Byte: byte b; if (byte.TryParse(text, out b)) return b; break; case TypeCode.Int16: short s; if (short.TryParse(text, out s)) return s; break; case TypeCode.UInt16: ushort us; if (ushort.TryParse(text, out us)) return us; break; case TypeCode.Int32: int i; if (int.TryParse(text, out i)) return i; break; case TypeCode.UInt32: uint ui; if (uint.TryParse(text, out ui)) return ui; break; case TypeCode.Int64: long l; if (long.TryParse(text, out l)) return l; break; case TypeCode.UInt64: ulong ul; if (ulong.TryParse(text, out ul)) return ul; break; case TypeCode.Single: float f; if (float.TryParse(text, out f)) return f; break; case TypeCode.Double: double d; if (double.TryParse(text, out d)) return d; break; case TypeCode.Decimal: decimal e; if (decimal.TryParse(text, out e)) return e; break; } return null; } static object ParseEnum(string name, Type type) { if (type.IsEnum) { MemberInfo[] memberInfos = type.FindMembers(MemberTypes.Field, BindingFlags.Public | BindingFlags.DeclaredOnly | BindingFlags.Static, Type.FilterNameIgnoreCase, name); if (memberInfos.Length != 0) return ((FieldInfo)memberInfos[0]).GetValue(null); } return null; } static bool IsCompatibleWith(Type source, Type target) { if (source == target) return true; if (!target.IsValueType) return target.IsAssignableFrom(source); Type st = GetNonNullableType(source); Type tt = GetNonNullableType(target); if (st != source && tt == target) return false; TypeCode sc = st.IsEnum ? TypeCode.Object : Type.GetTypeCode(st); TypeCode tc = tt.IsEnum ? TypeCode.Object : Type.GetTypeCode(tt); switch (sc) { case TypeCode.SByte: switch (tc) { case TypeCode.SByte: case TypeCode.Int16: case TypeCode.Int32: case TypeCode.Int64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.Byte: switch (tc) { case TypeCode.Byte: case TypeCode.Int16: case TypeCode.UInt16: case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.Int16: switch (tc) { case TypeCode.Int16: case TypeCode.Int32: case TypeCode.Int64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.UInt16: switch (tc) { case TypeCode.UInt16: case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.Int32: switch (tc) { case TypeCode.Int32: case TypeCode.Int64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.UInt32: switch (tc) { case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.Int64: switch (tc) { case TypeCode.Int64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.UInt64: switch (tc) { case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: return true; } break; case TypeCode.Single: switch (tc) { case TypeCode.Single: case TypeCode.Double: return true; } break; default: if (st == tt) return true; break; } return false; } static bool IsBetterThan(Expression[] args, MethodData m1, MethodData m2) { bool better = false; for (int i = 0; i < args.Length; i++) { int c = CompareConversions(args[i].Type, m1.Parameters[i].ParameterType, m2.Parameters[i].ParameterType); if (c < 0) return false; if (c > 0) better = true; } return better; } // Return 1 if s -> t1 is a better conversion than s -> t2 // Return -1 if s -> t2 is a better conversion than s -> t1 // Return 0 if neither conversion is better static int CompareConversions(Type s, Type t1, Type t2) { if (t1 == t2) return 0; if (s == t1) return 1; if (s == t2) return -1; bool t1t2 = IsCompatibleWith(t1, t2); bool t2t1 = IsCompatibleWith(t2, t1); if (t1t2 && !t2t1) return 1; if (t2t1 && !t1t2) return -1; if (IsSignedIntegralType(t1) && IsUnsignedIntegralType(t2)) return 1; if (IsSignedIntegralType(t2) && IsUnsignedIntegralType(t1)) return -1; return 0; } Expression GenerateEqual(Expression left, Expression right) { return Expression.Equal(left, right); } Expression GenerateNotEqual(Expression left, Expression right) { return Expression.NotEqual(left, right); } Expression GenerateGreaterThan(Expression left, Expression right) { if (left.Type == typeof(string)) { return Expression.GreaterThan( GenerateStaticMethodCall("Compare", left, right), Expression.Constant(0) ); } return Expression.GreaterThan(left, right); } Expression GenerateGreaterThanEqual(Expression left, Expression right) { if (left.Type == typeof(string)) { return Expression.GreaterThanOrEqual( GenerateStaticMethodCall("Compare", left, right), Expression.Constant(0) ); } return Expression.GreaterThanOrEqual(left, right); } Expression GenerateLessThan(Expression left, Expression right) { if (left.Type == typeof(string)) { return Expression.LessThan( GenerateStaticMethodCall("Compare", left, right), Expression.Constant(0) ); } return Expression.LessThan(left, right); } Expression GenerateLessThanEqual(Expression left, Expression right) { if (left.Type == typeof(string)) { return Expression.LessThanOrEqual( GenerateStaticMethodCall("Compare", left, right), Expression.Constant(0) ); } return Expression.LessThanOrEqual(left, right); } MethodInfo GetStaticMethod(string methodName, Expression left, Expression right) { return left.Type.GetMethod(methodName, new[] { left.Type, right.Type }); } Expression GenerateStaticMethodCall(string methodName, Expression left, Expression right) { return Expression.Call(null, GetStaticMethod(methodName, left, right), new[] { left, right }); } void SetTextPos(int pos) { textPos = pos; ch = textPos < textLen ? text[textPos] : '\0'; } void NextChar() { if (textPos < textLen) textPos++; ch = textPos < textLen ? text[textPos] : '\0'; } void NextToken() { while (Char.IsWhiteSpace(ch)) NextChar(); TokenId t; int tokenPos = textPos; switch (ch) { case '!': NextChar(); if (ch == '=') { NextChar(); t = TokenId.ExclamationEqual; } else { t = TokenId.Exclamation; } break; case '%': NextChar(); t = TokenId.Percent; break; case '&': NextChar(); if (ch == '&') { NextChar(); t = TokenId.DoubleAmphersand; } else { t = TokenId.Amphersand; } break; case '(': NextChar(); t = TokenId.OpenParen; break; case ')': NextChar(); t = TokenId.CloseParen; break; case '*': NextChar(); t = TokenId.Asterisk; break; case '+': NextChar(); t = TokenId.Plus; break; case ',': NextChar(); t = TokenId.Comma; break; case '-': NextChar(); t = TokenId.Minus; break; case '.': NextChar(); t = TokenId.Dot; break; case '/': NextChar(); t = TokenId.Slash; break; case ':': NextChar(); t = TokenId.Colon; break; case '<': NextChar(); if (ch == '=') { NextChar(); t = TokenId.LessThanEqual; } else if (ch == '>') { NextChar(); t = TokenId.LessGreater; } else { t = TokenId.LessThan; } break; case '=': NextChar(); if (ch == '=') { NextChar(); t = TokenId.DoubleEqual; } else { t = TokenId.Equal; } break; case '>': NextChar(); if (ch == '=') { NextChar(); t = TokenId.GreaterThanEqual; } else { t = TokenId.GreaterThan; } break; case '?': NextChar(); t = TokenId.Question; break; case '[': NextChar(); t = TokenId.OpenBracket; break; case ']': NextChar(); t = TokenId.CloseBracket; break; case '|': NextChar(); if (ch == '|') { NextChar(); t = TokenId.DoubleBar; } else { t = TokenId.Bar; } break; case '"': case '\'': char quote = ch; do { NextChar(); while (textPos < textLen && ch != quote) NextChar(); if (textPos == textLen) throw ParseError(textPos, Res.UnterminatedStringLiteral); NextChar(); } while (ch == quote); t = TokenId.StringLiteral; break; default: if (Char.IsLetter(ch) || ch == '@' || ch == '_') { do { NextChar(); } while (Char.IsLetterOrDigit(ch) || ch == '_'); t = TokenId.Identifier; break; } if (Char.IsDigit(ch)) { t = TokenId.IntegerLiteral; do { NextChar(); } while (Char.IsDigit(ch)); if (ch == '.') { t = TokenId.RealLiteral; NextChar(); ValidateDigit(); do { NextChar(); } while (Char.IsDigit(ch)); } if (ch == 'E' || ch == 'e') { t = TokenId.RealLiteral; NextChar(); if (ch == '+' || ch == '-') NextChar(); ValidateDigit(); do { NextChar(); } while (Char.IsDigit(ch)); } if (ch == 'F' || ch == 'f') NextChar(); break; } if (textPos == textLen) { t = TokenId.End; break; } throw ParseError(textPos, Res.InvalidCharacter, ch); } token.id = t; token.text = text.Substring(tokenPos, textPos - tokenPos); token.pos = tokenPos; } bool TokenIdentifierIs(string id) { return token.id == TokenId.Identifier && String.Equals(id, token.text, StringComparison.OrdinalIgnoreCase); } string GetIdentifier() { ValidateToken(TokenId.Identifier, Res.IdentifierExpected); string id = token.text; if (id.Length > 1 && id[0] == '@') id = id.Substring(1); return id; } void ValidateDigit() { if (!Char.IsDigit(ch)) throw ParseError(textPos, Res.DigitExpected); } void ValidateToken(TokenId t, string errorMessage) { if (token.id != t) throw ParseError(errorMessage); } void ValidateToken(TokenId t) { if (token.id != t) throw ParseError(Res.SyntaxError); } Exception ParseError(string format, params object[] args) { return ParseError(token.pos, format, args); } Exception ParseError(int pos, string format, params object[] args) { return new ParseException(string.Format(System.Globalization.CultureInfo.CurrentCulture, format, args), pos); } static Dictionary CreateKeywords() { Dictionary d = new Dictionary(StringComparer.OrdinalIgnoreCase); d.Add("true", trueLiteral); d.Add("false", falseLiteral); d.Add("null", nullLiteral); //d.Add(keywordIt, keywordIt); //d.Add(keywordIif, keywordIif); //d.Add(keywordNew, keywordNew); foreach (Type type in predefinedTypes) d.Add(type.Name, type); return d; } } static class Res { public const string DuplicateIdentifier = "The identifier '{0}' was defined more than once"; public const string ExpressionTypeMismatch = "Expression of type '{0}' expected"; public const string ExpressionExpected = "Expression expected"; public const string InvalidCharacterLiteral = "Character literal must contain exactly one character"; public const string InvalidIntegerLiteral = "Invalid integer literal '{0}'"; public const string InvalidRealLiteral = "Invalid real literal '{0}'"; public const string UnknownIdentifier = "Unknown identifier '{0}'"; public const string NoItInScope = "No 'it' is in scope"; public const string IifRequiresThreeArgs = "The 'iif' function requires three arguments"; public const string FirstExprMustBeBool = "The first expression must be of type 'Boolean'"; public const string BothTypesConvertToOther = "Both of the types '{0}' and '{1}' convert to the other"; public const string NeitherTypeConvertsToOther = "Neither of the types '{0}' and '{1}' converts to the other"; public const string MissingAsClause = "Expression is missing an 'as' clause"; public const string ArgsIncompatibleWithLambda = "Argument list incompatible with lambda expression"; public const string TypeHasNoNullableForm = "Type '{0}' has no nullable form"; public const string NoMatchingConstructor = "No matching constructor in type '{0}'"; public const string AmbiguousConstructorInvocation = "Ambiguous invocation of '{0}' constructor"; public const string CannotConvertValue = "A value of type '{0}' cannot be converted to type '{1}'"; public const string NoApplicableMethod = "No applicable method '{0}' exists in type '{1}'"; public const string MethodsAreInaccessible = "Methods on type '{0}' are not accessible"; public const string MethodIsVoid = "Method '{0}' in type '{1}' does not return a value"; public const string AmbiguousMethodInvocation = "Ambiguous invocation of method '{0}' in type '{1}'"; public const string UnknownPropertyOrField = "No property or field '{0}' exists in type '{1}'"; public const string NoApplicableAggregate = "No applicable aggregate method '{0}' exists"; public const string CannotIndexMultiDimArray = "Indexing of multi-dimensional arrays is not supported"; public const string InvalidIndex = "Array index must be an integer expression"; public const string NoApplicableIndexer = "No applicable indexer exists in type '{0}'"; public const string AmbiguousIndexerInvocation = "Ambiguous invocation of indexer in type '{0}'"; public const string IncompatibleOperand = "Operator '{0}' incompatible with operand type '{1}'"; public const string IncompatibleOperands = "Operator '{0}' incompatible with operand types '{1}' and '{2}'"; public const string UnterminatedStringLiteral = "Unterminated string literal"; public const string InvalidCharacter = "Syntax error '{0}'"; public const string DigitExpected = "Digit expected"; public const string SyntaxError = "Syntax error"; public const string TokenExpected = "{0} expected"; public const string ParseExceptionFormat = "{0} (at index {1})"; public const string ColonExpected = "':' expected"; public const string OpenParenExpected = "'(' expected"; public const string CloseParenOrOperatorExpected = "')' or operator expected"; public const string CloseParenOrCommaExpected = "')' or ',' expected"; public const string DotOrOpenParenExpected = "'.' or '(' expected"; public const string OpenBracketExpected = "'[' expected"; public const string CloseBracketOrCommaExpected = "']' or ',' expected"; public const string IdentifierExpected = "Identifier expected"; } } ================================================ FILE: RaptorDB/Views/LINQQuery.cs ================================================ using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; namespace RaptorDB.Views { delegate MGRB QueryFromTo(string colname, object from, object to); delegate MGRB QueryExpression(string colname, RDBExpression exp, object from); internal class QueryVisitor : ExpressionVisitor { public QueryVisitor(QueryExpression express, QueryFromTo fromto) { qexpression = express; qfromto = fromto; } public Stack _stack = new Stack(); public Stack _bitmap = new Stack(); QueryFromTo qfromto; QueryExpression qexpression; private bool _leftmode = true; protected override Expression VisitBinary(BinaryExpression b) { _leftmode = true; var m = this.Visit(b.Left); if (m == null) // VB.net sty;e linq for string compare return b.Right; ExpressionType t = b.NodeType; if (t == ExpressionType.Equal || t == ExpressionType.NotEqual || t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual || t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual) _stack.Push(b.NodeType); _leftmode = false; this.Visit(b.Right); t = b.NodeType; if (t == ExpressionType.Equal || t == ExpressionType.NotEqual || t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan || t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan ) { // binary expression object lval = _stack.Pop(); ExpressionType lop = (ExpressionType)_stack.Pop(); string lname = (string)_stack.Pop(); if (_stack.Count > 0) { lname += "_" + (string)_stack.Pop(); } RDBExpression exp = RDBExpression.Equal; if (lop == ExpressionType.LessThan) exp = RDBExpression.Less; else if (lop == ExpressionType.LessThanOrEqual) exp = RDBExpression.LessEqual; else if (lop == ExpressionType.GreaterThan) exp = RDBExpression.Greater; else if (lop == ExpressionType.GreaterThanOrEqual) exp = RDBExpression.GreaterEqual; else if (lop == ExpressionType.NotEqual) exp = RDBExpression.NotEqual; _bitmap.Push(qexpression(lname, exp, lval)); } if (t == ExpressionType.And || t == ExpressionType.AndAlso || t == ExpressionType.Or || t == ExpressionType.OrElse) { if (_bitmap.Count > 1) { // do bitmap operations MGRB right = (MGRB)_bitmap.Pop(); MGRB left = (MGRB)_bitmap.Pop(); if (t == ExpressionType.And || t == ExpressionType.AndAlso) _bitmap.Push(right.And(left)); if (t == ExpressionType.Or || t == ExpressionType.OrElse) _bitmap.Push(right.Or(left)); } else { // single bitmap operation } } return b; } protected override Expression VisitMethodCall(MethodCallExpression m) { string s = m.ToString(); // FEATURE : add contains , startswith ?? // VB.net : e.g. CompareString(x.NoCase, "Me 4", False) if (s.StartsWith("CompareString")) { var left = m.Arguments[0]; // Removes dot if any var leftStr = left.ToString().Substring(left.ToString().IndexOf('.') + 1); var right = m.Arguments[1].ToString().Replace("\"", String.Empty); RDBExpression exp = RDBExpression.Equal; _bitmap.Push(qexpression("" + leftStr, exp, right)); return null; } string mc = s.Substring(s.IndexOf('.') + 1); if (mc.Contains("Between")) { Type datatype = m.Arguments[0].Type; var ss = m.Arguments[0].ToString().Split('.'); string name = ""; if (ss.Length > 2) { // handle datetype.year etc name = ss[1] + "_$" + ss[2]; } else name = ss[1]; if (datatype == typeof(DateTime)) { DateTime from = DateTime.Now; DateTime to = DateTime.Now; if (m.Arguments[1].Type == typeof(string)) { from = DateTime.Parse((string)GetValueForMember(m.Arguments[1])); to = DateTime.Parse((string)GetValueForMember(m.Arguments[2])); } else { from = (DateTime)GetValueForMember(m.Arguments[1]); to = (DateTime)GetValueForMember(m.Arguments[2]); } _bitmap.Push(qfromto(name, from, to)); } else { var from = GetValueForMember(m.Arguments[1]); var to = GetValueForMember(m.Arguments[2]); _bitmap.Push(qfromto(name, from, to)); } } else if (mc.Contains("In")) { var ss = m.Arguments[0].ToString().Split('.'); string name = ""; if (ss.Length > 2) { // handle datetype.year etc name = ss[1] + "_$" + ss[2]; } else name = ss[1]; _InCommand = name; Visit(m.Arguments[1]); _bitmap.Push(_inBmp); _inBmp = null; _InCommand = ""; } else _stack.Push(mc); return m; } private MGRB _inBmp = null; private string _InCommand = ""; private int _count = 0; public override Expression Visit(Expression node) { if (node.NodeType == ExpressionType.NewArrayInit) { var a = node as NewArrayExpression; _count = a.Expressions.Count; return base.Visit(node); } else if (node.NodeType == ExpressionType.MemberAccess) { var v = base.Visit(node); if (_InCommand != "") { var a = _stack.Pop() as IList; foreach (var c in a) { if (_inBmp == null) _inBmp = qexpression(_InCommand, RDBExpression.Equal, c); else _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c)); } } return v; } else return base.Visit(node); } private object GetValueForMember(object m) { object val = null; var f = m as ConstantExpression; if (f != null) return f.Value; var mm = m as MemberExpression; if (mm.NodeType == ExpressionType.MemberAccess) { Type tt = mm.Expression.Type; val = tt.InvokeMember(mm.Member.Name, BindingFlags.GetField | BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, (mm.Expression as ConstantExpression).Value, null); } return val; } protected override Expression VisitMember(MemberExpression m) { var n = m.Member.Name; if (n != null && m.Expression.Type == typeof(DateTime)) { _stack.Push("$" + n); } var e = base.VisitMember(m); var c = m.Expression as ConstantExpression; if (c != null) { Type t = c.Value.GetType(); var x = t.InvokeMember(m.Member.Name, BindingFlags.GetField | BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, c.Value, null); _stack.Push(x); } if (m.Expression != null) { if (m.Expression.NodeType == ExpressionType.Parameter) // property _stack.Push(m.Member.Name); else if (m.Expression.NodeType == ExpressionType.MemberAccess) // obj.property { if (_leftmode == false) { Type t = m.Expression.Type; var val = t.InvokeMember(m.Member.Name, BindingFlags.GetField | BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, _stack.Pop(), null); _stack.Push(val); } } } return e; } protected override Expression VisitConstant(ConstantExpression c) { IQueryable q = c.Value as IQueryable; if (q != null) { _stack.Push(q.ElementType.Name); } else if (c.Value == null) { _stack.Push(null); } else { Type t = c.Value.GetType(); if (t.IsValueType || t == typeof(string)) { if (_InCommand != "") { if (_inBmp == null) _inBmp = qexpression(_InCommand, RDBExpression.Equal, c.Value); else _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c.Value)); } else _stack.Push(c.Value); } } return c; } } } ================================================ FILE: RaptorDB/Views/TaskQueue.cs ================================================ using System; using System.Timers; using System.Threading.Tasks; using System.Collections.Concurrent; namespace RaptorDB.Views { internal class TaskQueue { public TaskQueue() { _timer = new Timer(); _timer.Interval = Global.TaskCleanupTimerSeconds * 1000; _timer.Elapsed += new ElapsedEventHandler(_timer_Elapsed); _timer.Enabled = true; _log.Debug("TaskQueue starting"); } private ILog _log = LogManager.GetLogger(typeof(TaskQueue)); private object _lock = new object(); private bool _shuttingdown = false; private Timer _timer; private ConcurrentQueue _que = new ConcurrentQueue(); void _timer_Elapsed(object sender, ElapsedEventArgs e) { lock (_lock) { while (_que.Count > 0) { //_log.Debug("in queue cleanup, count = " + _que.Count); if (_shuttingdown) break; Task t = null; if (_que.TryPeek(out t)) { if (t.IsCompleted || t.IsCanceled || t.IsFaulted) _que.TryDequeue(out t); else break; } else break; } } } public void AddTask(Action action) { if (_shuttingdown == false) _que.Enqueue(Task.Factory.StartNew(action)); } public void Shutdown() { _log.Debug("TaskQueue shutdown"); // wait for tasks to finish _shuttingdown = true; _timer.Enabled = false; if (_que.Count > 0) Task.WaitAll(_que.ToArray()); } } } ================================================ FILE: RaptorDB/Views/ViewHandler.cs ================================================ using fastBinaryJSON; using fastJSON; using RaptorDB.Common; using System; using System.Collections.Generic; using System.IO; using System.Linq.Expressions; using System.Reflection; using System.Reflection.Emit; using System.Text; using System.Threading; using System.Threading.Tasks; namespace RaptorDB.Views { public class ViewRowDefinition { public ViewRowDefinition() { Columns = new List>(); } public List> Columns { get; set; } public void Add(string name, Type type) { Columns.Add(new KeyValuePair(name, type)); } } internal class tran_data { public Guid docid; public Dictionary> rows; } internal class ViewHandler { private ILog _log = LogManager.GetLogger(typeof(ViewHandler)); public ViewHandler(string path, ViewManager manager) { _Path = path; _viewmanager = manager; } private string _S = Path.DirectorySeparatorChar.ToString(); private string _Path = ""; private ViewManager _viewmanager; internal ViewBase _view; private Dictionary _indexes = new Dictionary(); private StorageFile _viewData; private BoolIndex _deletedRows; private string _docid = "docid"; private List _colnames = new List(); private RowFill _rowfiller; private ViewRowDefinition _schema; private ViewRowDefinition _datecolumns; private SafeDictionary _transactions = new SafeDictionary(); private SafeDictionary _nocase = new SafeDictionary(); private Dictionary _idxlen = new Dictionary(); private System.Timers.Timer _saveTimer; Type basetype; // used for mapper dynamic _mapper; bool _isDirty = false; private string _dirtyFilename = "temp.$"; private bool _stsaving = false; private int _RaptorDBVersion = 5; // used for engine changes to views private string _RaptorDBVersionFilename = "RaptorDB.version"; // TODO : showing incorrect results //private SafeDictionary _queryCache = new SafeDictionary(); private object _savetimerlock = new object(); void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { lock (_savetimerlock) { _stsaving = true; foreach (var i in _indexes) i.Value.SaveIndex(); _deletedRows.SaveIndex(); _stsaving = false; } } public Type GetFireOnType() { return basetype; } internal void SetView(View view, IDocStorage docs) { bool rebuild = false; _view = view; // generate schemacolumns from schema GenerateSchemaColumns(_view); if (_Path.EndsWith(_S) == false) _Path += _S; _Path += view.Name + _S; if (Directory.Exists(_Path) == false) { Directory.CreateDirectory(_Path); rebuild = true; } else { // read version file and check with view int version = 0; if (File.Exists(_Path + _view.Name + ".version")) { int.TryParse(File.ReadAllText(_Path + _view.Name + ".version"), out version); if (version != view.Version) { _log.Debug("Newer view version detected"); rebuild = true; } } } if (File.Exists(_Path + _dirtyFilename)) { _log.Debug("Last shutdown failed, rebuilding view : " + _view.Name); rebuild = true; } if (File.Exists(_Path + _RaptorDBVersionFilename)) { // check view engine version string s = File.ReadAllText(_Path + _RaptorDBVersionFilename); int version = 0; int.TryParse(s, out version); if (version != _RaptorDBVersion) { _log.Debug("RaptorDB view engine upgrade, rebuilding view : " + _view.Name); rebuild = true; } } else { _log.Debug("RaptorDB view engine upgrade, rebuilding view : " + _view.Name); rebuild = true; } if (rebuild) { _log.Debug("Deleting old view data folder = " + view.Name); Directory.Delete(_Path, true); Directory.CreateDirectory(_Path); } // load indexes here CreateLoadIndexes(); _deletedRows = new BoolIndex(_Path, _view.Name, ".deleted"); _viewData = new StorageFile(_Path + view.Name + ".mgdat"); CreateResultRowFiller(); _mapper = view.Mapper; // looking for the T in View if (view.GetType().GetGenericArguments().Length == 1) // HACK : kludge change when possible basetype = view.GetType().GetGenericArguments()[0]; else { // or recurse until found basetype = view.GetType().BaseType.GetGenericArguments()[0]; } if (rebuild) Task.Factory.StartNew(() => RebuildFromScratch(docs)); _saveTimer = new System.Timers.Timer(); _saveTimer.AutoReset = true; _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed); _saveTimer.Interval = Global.SaveIndexToDiskTimerSeconds * 1000; _saveTimer.Start(); } internal void FreeMemory() { _log.Debug("free memory : " + _view.Name); foreach (var i in _indexes) i.Value.FreeMemory(); _deletedRows.FreeMemory(); InvalidateCaches(); } internal void Commit(int ID) { tran_data data = null; // save data to indexes if (_transactions.TryGetValue(ID, out data)) { // delete any items with docid in view if (_view.DeleteBeforeInsert) DeleteRowsWith(data.docid); SaveAndIndex(data.rows); } // remove in memory data _transactions.Remove(ID); } internal void RollBack(int ID) { // remove in memory data _transactions.Remove(ID); } internal void Insert(Guid guid, T doc) { apimapper api = new apimapper(_viewmanager, this); if (basetype == doc.GetType()) { View view = (View)_view; if (view.Mapper != null) view.Mapper(api, guid, doc); } else if (_mapper != null) _mapper(api, guid, doc); // map objects to rows foreach (var d in api.emitobj) api.emit.Add(d.Key, ExtractRows(d.Value)); // delete any items with docid in view if (_view.DeleteBeforeInsert) DeleteRowsWith(guid); SaveAndIndex(api.emit); } private void SaveAndIndex(Dictionary> rows) { foreach (var d in rows) { // insert new items into view InsertRowsWithIndexUpdate(d.Key, d.Value); } InvalidateCaches(); } internal bool InsertTransaction(Guid docid, T doc) { apimapper api = new apimapper(_viewmanager, this); if (basetype == doc.GetType()) { View view = (View)_view; try { if (view.Mapper != null) view.Mapper(api, docid, doc); } catch (Exception ex) { _log.Error(ex); return false; } } else if (_mapper != null) _mapper(api, docid, doc); if (api._RollBack == true) return false; // map emitobj -> rows foreach (var d in api.emitobj) api.emit.Add(d.Key, ExtractRows(d.Value)); tran_data data = new tran_data(); if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data)) { // TODO : exists -> merge data?? } else { data = new tran_data(); data.docid = docid; data.rows = api.emit; _transactions.Add(Thread.CurrentThread.ManagedThreadId, data); } return true; } internal Result Query(string filter, int start, int count, string orderby) { filter = filter.Trim(); if (filter == "") return Query(start, count, orderby); DateTime dt = FastDateTime.Now; _log.Debug("query : " + _view.Name); _log.Debug("query : " + filter); if (orderby != "") _log.Debug("orderby : " + orderby); var ba = GenerateBitmap(filter); var order = SortBy(orderby); bool desc = false; if (orderby.ToLower().Contains(" desc")) desc = true; _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); dt = FastDateTime.Now; // exec query return rows return ReturnRowsTyped(ba, null, start, count, order, desc); } internal Result Query(int start, int count) { return Query(start, count, ""); } internal Result Query(int start, int count, string orderby) { // no filter query -> just show all the data DateTime dt = FastDateTime.Now; _log.Debug("query : " + _view.Name); int totalviewrows = _viewData.Count(); List rows = new List(); Result ret = new Result(); int skip = start; int cc = 0; MGRB del = _deletedRows.GetBits(); ret.TotalCount = totalviewrows - (int)del.CountOnes(); var order = SortBy(orderby); bool desc = false; if (orderby.ToLower().Contains(" desc")) desc = true; if (order.Count == 0) for (int i = 0; i < totalviewrows; i++) order.Add(i); if (count == -1) count = totalviewrows; int len = order.Count; if (desc == false) { for (int idx = 0; idx < len; idx++) { extractrowobject(count, rows, ref skip, ref cc, del, order, idx); if (cc == count) break; } } else { for (int idx = len - 1; idx >= 0; idx--) { extractrowobject(count, rows, ref skip, ref cc, del, order, idx); if (cc == count) break; } } _log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); _log.Debug("query rows count : " + rows.Count.ToString("#,0")); ret.OK = true; ret.Count = rows.Count; ret.Rows = rows; return ret; } private void extractrowobject(int count, List rows, ref int skip, ref int cc, MGRB del, List order, int idx) { int i = order[idx]; if (del.Get(i) == false) { if (skip > 0) skip--; else { bool b = OutputRow(rows, i); if (b && count > 0) cc++; } } } internal void Shutdown() { try { lock (_savetimerlock) _saveTimer.Enabled = false; while (_stsaving) Thread.Sleep(1); if (_rebuilding) _log.Debug("Waiting for view rebuild to finish... : " + _view.Name); while (_rebuilding) Thread.Sleep(50); _log.Debug("Shutting down Viewhandler"); // shutdown indexes List tasks = new List(); foreach (var v in _indexes) { tasks.Add(Task.Factory.StartNew(() => { _log.Debug("Shutting down view index : " + v.Key); v.Value.Shutdown(); })); } Task.WaitAll(tasks.ToArray()); // save deletedbitmap _deletedRows.Shutdown(); _viewData.Shutdown(); // write view version File.WriteAllText(_Path + _view.Name + ".version", _view.Version.ToString()); File.WriteAllText(_Path + _RaptorDBVersionFilename, _RaptorDBVersion.ToString()); // remove dirty file if (File.Exists(_Path + _dirtyFilename)) File.Delete(_Path + _dirtyFilename); _log.Debug("Viewhandler shutdown done."); } catch (Exception ex) { _log.Error(ex); } } internal void Delete(Guid docid) { DeleteRowsWith(docid); InvalidateCaches(); } #region [ private methods ] private void CreateResultRowFiller() { _rowfiller = CreateRowFillerDelegate(_view.Schema, _schema); // init the row create _createrow = null; FastCreateObject(_view.Schema); } public delegate object RowFill(object o, object[] data); public static RowFill CreateRowFillerDelegate(Type objtype, ViewRowDefinition schema) { DynamicMethod dynMethod = new DynamicMethod("rowfill", typeof(object), new Type[] { typeof(object), typeof(object[]) }); ILGenerator il = dynMethod.GetILGenerator(); var row = il.DeclareLocal(objtype); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Castclass, objtype); il.Emit(OpCodes.Stloc, row); int i = 1; var val = il.DeclareLocal(typeof(object)); var fields = objtype.GetFields(); var properties = objtype.GetProperties(); foreach (var col in schema.Columns) { FieldInfo c = null; PropertyInfo p = null; if (isField(fields, col.Key, out c)) { var end = il.DefineLabel(); il.Emit(OpCodes.Ldarg_1); if (col.Key != "docid") il.Emit(OpCodes.Ldc_I4, i); else il.Emit(OpCodes.Ldc_I4, 0); il.Emit(OpCodes.Ldelem_Ref); // check if value is not null il.Emit(OpCodes.Stloc, val); il.Emit(OpCodes.Ldloc, val); il.Emit(OpCodes.Brfalse_S, end); il.Emit(OpCodes.Ldloc, row); il.Emit(OpCodes.Ldloc, val); il.Emit(OpCodes.Unbox_Any, c.FieldType); il.Emit(OpCodes.Stfld, c); il.MarkLabel(end); i++; } else if (isProperty(properties, col.Key, out p)) { var end = il.DefineLabel(); MethodInfo setMethod = p.GetSetMethod(); il.Emit(OpCodes.Ldarg_1); if (col.Key != "docid") il.Emit(OpCodes.Ldc_I4, i); else il.Emit(OpCodes.Ldc_I4, 0); il.Emit(OpCodes.Ldelem_Ref); // check if value is not null il.Emit(OpCodes.Stloc, val); il.Emit(OpCodes.Ldloc, val); il.Emit(OpCodes.Brfalse_S, end); il.Emit(OpCodes.Ldloc, row); il.Emit(OpCodes.Ldloc, val); il.Emit(OpCodes.Unbox_Any, p.PropertyType); if (!p.DeclaringType.IsValueType) il.EmitCall(OpCodes.Callvirt, setMethod, null); else il.EmitCall(OpCodes.Call, setMethod, null); il.MarkLabel(end); i++; } } il.Emit(OpCodes.Ldloc, row); il.Emit(OpCodes.Ret); return (RowFill)dynMethod.CreateDelegate(typeof(RowFill)); } private static bool isProperty(PropertyInfo[] properties, string key, out PropertyInfo p) { foreach (var i in properties) if (i.Name == key) { p = i; return true; } p = null; return false; } private static bool isField(FieldInfo[] fields, string key, out FieldInfo c) { foreach (var i in fields) if (i.Name == key) { c = i; return true; } c = null; return false; } //private Result ReturnRowsObject(MGRB ba, List trows, int start, int count, List orderby, bool descending) //{ // DateTime dt = FastDateTime.Now; // List rows = new List(); // Result ret = new Result(); // int skip = start; // int c = 0; // ret.TotalCount = (int)ba.CountOnes(); // if (count == -1) count = ret.TotalCount; // if (count > 0) // { // int len = orderby.Count; // if (len > 0) // { // if (descending == false) // { // for (int idx = 0; idx < len; idx++) // { // extractsortrowobject(ba, count, orderby, rows, ref skip, ref c, idx); // if (c == count) break; // } // } // else // { // for (int idx = len - 1; idx >= 0; idx--) // { // extractsortrowobject(ba, count, orderby, rows, ref skip, ref c, idx); // if (c == count) break; // } // } // } // foreach (int i in ba.GetBitIndexes()) // { // if (c < count) // { // if (skip > 0) // skip--; // else // { // bool b = OutputRow(rows, i); // if (b && count > 0) // c++; // } // if (c == count) break; // } // } // } // if (trows != null) // TODO : move to start and decrement in count // foreach (var o in trows) // rows.Add(o); // _log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); // _log.Debug("query rows count : " + rows.Count.ToString("#,0")); // ret.OK = true; // ret.Count = rows.Count; // ret.Rows = rows; // return ret; //} private void extractsortrowobject(MGRB ba, int count, List orderby, List rows, ref int skip, ref int c, int idx) { int i = orderby[idx]; if (ba.Get(i)) { if (skip > 0) skip--; else { bool b = OutputRow(rows, i); if (b && count > 0) c++; } ba.Set(i, false); } } private bool OutputRow(List rows, int i) { byte[] b = _viewData.ViewReadRawBytes(i); if (b != null) { object o = FastCreateObject(_view.Schema); object[] data = (object[])BJSON.ToObject(b); rows.Add((T)_rowfiller(o, data)); return true; } return false; } private Result ReturnRowsTyped(MGRB ba, List trows, int start, int count, List orderby, bool descending) { DateTime dt = FastDateTime.Now; List rows = new List(); Result ret = new Result(); int skip = start; int c = 0; ret.TotalCount = (int)ba.CountOnes(); if (count == -1) count = ret.TotalCount; if (count > 0) { int len = orderby.Count; if (len > 0) { if (descending == false) { for (int idx = 0; idx < len; idx++) //foreach (int i in orderby) { extractsortrowT(ba, count, orderby, rows, ref skip, ref c, idx); if (c == count) break; } } else { for (int idx = len - 1; idx >= 0; idx--) //foreach (int i in orderby) { extractsortrowT(ba, count, orderby, rows, ref skip, ref c, idx); if (c == count) break; } } } foreach (int i in ba.GetBitIndexes()) { if (c < count) { if (skip > 0) skip--; else { bool b = OutputRow(rows, i); if (b && count > 0) c++; } if (c == count) break; } } } if (trows != null)// TODO : move to start and decrement in count foreach (var o in trows) rows.Add(o); _log.Debug("query rows fetched (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); _log.Debug("query rows count : " + rows.Count.ToString("#,0")); ret.OK = true; ret.Count = rows.Count; ret.Rows = rows; return ret; } private void extractsortrowT(MGRB ba, int count, List orderby, List rows, ref int skip, ref int c, int idx) { int i = orderby[idx]; if (ba.Get(i)) { if (skip > 0) skip--; else { bool b = OutputRow(rows, i); if (b && count > 0) c++; } ba.Set(i, false); } } private CreateRow _createrow = null; private delegate object CreateRow(); private object FastCreateObject(Type objtype) { try { if (_createrow != null) return _createrow(); else { DynamicMethod dynMethod = new DynamicMethod("_", objtype, null); ILGenerator ilGen = dynMethod.GetILGenerator(); ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(Type.EmptyTypes)); ilGen.Emit(OpCodes.Ret); _createrow = (CreateRow)dynMethod.CreateDelegate(typeof(CreateRow)); return _createrow(); } } catch (Exception exc) { throw new Exception(string.Format("Failed to fast create instance for type '{0}' from assemebly '{1}'", objtype.FullName, objtype.AssemblyQualifiedName), exc); } } MethodInfo _insertmethod = null; bool _rebuilding = false; private void RebuildFromScratch(IDocStorage docs) { _rebuilding = true; try { _insertmethod = this.GetType().GetMethod("Insert", BindingFlags.Instance | BindingFlags.NonPublic); _log.Debug("Rebuilding view from scratch..."); _log.Debug("View = " + _view.Name); DateTime dt = FastDateTime.Now; int c = docs.RecordCount(); int dc = 0; for (int i = 0; i < c; i++) { StorageItem meta = null; object b = docs.GetObject(i, out meta); if (meta != null && meta.isDeleted) Delete(meta.key); else { if (b != null) { object obj = b; Type t = obj.GetType(); if (t == typeof(View_delete)) { View_delete vd = (View_delete)obj; if (vd.Viewname.ToLower() == this._view.Name.ToLower()) ViewDelete(vd.Filter); } else if (t == typeof(View_insert)) { View_insert vi = (View_insert)obj; if (vi.Viewname.ToLower() == this._view.Name.ToLower()) ViewInsert(vi.ID, vi.RowObject); } else if (t.IsSubclassOf(basetype) || t == basetype) { var m = _insertmethod.MakeGenericMethod(new Type[] { obj.GetType() }); m.Invoke(this, new object[] { meta.key, obj }); dc++; } } else _log.Error("Doc is null : " + meta.key); } } _log.Debug("Documents processed = " + dc); _log.Debug("rebuild view '" + _view.Name + "' done (s) = " + FastDateTime.Now.Subtract(dt).TotalSeconds); // write version.dat file when done File.WriteAllText(_Path + _view.Name + ".version", _view.Version.ToString()); } catch (Exception ex) { _log.Error("Rebuilding View failed : " + _view.Name, ex); } _rebuilding = false; InvalidateCaches(); } private object CreateObject(byte[] b) { if (b[0] < 32) return BJSON.ToObject(b); else return JSON.ToObject(Encoding.ASCII.GetString(b)); } private void CreateLoadIndexes() { int i = 0; _indexes.Add(_docid, new TypeIndexes(_Path, _docid, 16)); // load indexes foreach (var c in _schema.Columns) { if (c.Key != "docid") _indexes.Add(_schema.Columns[i].Key, CreateIndex( _schema.Columns[i].Key, _schema.Columns[i].Value)); i++; } i = 0; foreach (var c in _datecolumns.Columns) { _indexes.Add(_datecolumns.Columns[i].Key, CreateIndex( _datecolumns.Columns[i].Key, _datecolumns.Columns[i].Value)); i++; } } private void GenerateSchemaColumns(ViewBase _view) { // generate schema columns from schema _schema = new ViewRowDefinition(); _datecolumns = new ViewRowDefinition(); foreach (var p in _view.Schema.GetProperties()) { Type t = p.PropertyType; if (_view.NoIndexingColumns.Contains(p.Name) || _view.NoIndexingColumns.Contains(p.Name.ToLower())) { _schema.Add(p.Name, typeof(NoIndexing)); } else { if (p.GetCustomAttributes(typeof(FullTextAttribute), true).Length > 0) t = typeof(FullTextString); if (_view.FullTextColumns.Contains(p.Name) || _view.FullTextColumns.Contains(p.Name.ToLower())) t = typeof(FullTextString); if (p.Name != "docid") _schema.Add(p.Name, t); if (p.GetCustomAttributes(typeof(CaseInsensitiveAttribute), true).Length > 0) _nocase.Add(p.Name, 0); if (_view.CaseInsensitiveColumns.Contains(p.Name) || _view.CaseInsensitiveColumns.Contains(p.Name.ToLower())) _nocase.Add(p.Name, 0); if (t != typeof(FullTextString)) { var a = p.GetCustomAttributes(typeof(StringIndexLength), false); if (a.Length > 0) { byte l = (a[0] as StringIndexLength).Length; _idxlen.Add(p.Name, l); } if (_view.StringIndexLength.ContainsKey(p.Name) || _view.StringIndexLength.ContainsKey(p.Name.ToLower())) { byte b = 0; if (_view.StringIndexLength.TryGetValue(p.Name, out b)) _idxlen.Add(p.Name, b); if (_view.StringIndexLength.TryGetValue(p.Name.ToLower(), out b)) _idxlen.Add(p.Name, b); } } if (t == typeof(DateTime)) { _datecolumns.Add(p.Name + "_$Year", typeof(int)); _datecolumns.Add(p.Name + "_$Month", typeof(int)); _datecolumns.Add(p.Name + "_$Day", typeof(int)); _datecolumns.Add(p.Name + "_$Hour", typeof(int)); _datecolumns.Add(p.Name + "_$Minute", typeof(int)); } } } foreach (var f in _view.Schema.GetFields()) { Type t = f.FieldType; if (_view.NoIndexingColumns.Contains(f.Name) || _view.NoIndexingColumns.Contains(f.Name.ToLower())) { _schema.Add(f.Name, typeof(NoIndexing)); } else { if (f.GetCustomAttributes(typeof(FullTextAttribute), true).Length > 0) t = typeof(FullTextString); if (_view.FullTextColumns.Contains(f.Name) || _view.FullTextColumns.Contains(f.Name.ToLower())) t = typeof(FullTextString); if (f.Name != "docid") _schema.Add(f.Name, t); if (f.GetCustomAttributes(typeof(CaseInsensitiveAttribute), true).Length > 0) _nocase.Add(f.Name, 0); if (_view.CaseInsensitiveColumns.Contains(f.Name) || _view.CaseInsensitiveColumns.Contains(f.Name.ToLower())) _nocase.Add(f.Name, 0); if (t != typeof(FullTextString)) { var a = f.GetCustomAttributes(typeof(StringIndexLength), false); if (a.Length > 0) { byte l = (a[0] as StringIndexLength).Length; _idxlen.Add(f.Name, l); } if (_view.StringIndexLength.ContainsKey(f.Name) || _view.StringIndexLength.ContainsKey(f.Name.ToLower())) { byte b = 0; if (_view.StringIndexLength.TryGetValue(f.Name, out b)) _idxlen.Add(f.Name, b); if (_view.StringIndexLength.TryGetValue(f.Name.ToLower(), out b)) _idxlen.Add(f.Name, b); } } if (t == typeof(DateTime)) { _datecolumns.Add(f.Name + "_$Year", typeof(int)); _datecolumns.Add(f.Name + "_$Month", typeof(int)); _datecolumns.Add(f.Name + "_$Day", typeof(int)); _datecolumns.Add(f.Name + "_$Hour", typeof(int)); _datecolumns.Add(f.Name + "_$Minute", typeof(int)); } } } _schema.Add("docid", typeof(Guid)); foreach (var s in _schema.Columns) _colnames.Add(s.Key); // set column index for nocase for (int i = 0; i < _colnames.Count; i++) { int j = 0; if (_nocase.TryGetValue(_colnames[i], out j)) _nocase[_colnames[i]] = i; } } private void InsertRowsWithIndexUpdate(Guid guid, List rows) { if (_isDirty == false) WriteDirtyFile(); foreach (var row in rows) { object[] r = new object[row.Length + 1]; r[0] = guid; Array.Copy(row, 0, r, 1, row.Length); byte[] b = BJSON.ToBJSON(r, new BJSONParameters { UseExtensions = false, UseTypedArrays = false }); int rownum = (int)_viewData.WriteRawData(b); // case insensitve columns here foreach (var kv in _nocase) row[kv.Value] = ("" + row[kv.Value]).ToLowerInvariant(); IndexRow(guid, row, rownum); } } private List ExtractRows(List rows) { List output = new List(); // reflection match object properties to the schema row int colcount = _schema.Columns.Count; foreach (var obj in rows) { object[] r = new object[colcount]; Getters[] getters = Reflection.Instance.GetGetters(obj.GetType(), /*true,*/ null); for (int i = 0; i < colcount; i++) { var c = _schema.Columns[i]; foreach (var g in getters) { if (g.Name == c.Key) { r[i] = g.Getter(obj); break; } } } output.Add(r); } return output; } private void IndexRow(Guid docid, object[] row, int rownum) { int c = _colnames.Count - 1; // skip last docid _indexes[_docid].Set(docid, rownum); for (int i = 0; i < c; i++) { string colname = _colnames[i]; object d = row[i]; var idx = _indexes[colname]; if (idx != null) idx.Set(d, rownum); foreach (var dc in _datecolumns.Columns) { if (dc.Key.ToLower().StartsWith(colname.ToLower() + "_$")) { DateTime dt = (DateTime)d; _indexes[colname + "_$Year"].Set(dt.Year, rownum); _indexes[colname + "_$Month"].Set(dt.Month, rownum); _indexes[colname + "_$Day"].Set(dt.Day, rownum); _indexes[colname + "_$Hour"].Set(dt.Hour, rownum); _indexes[colname + "_$Minute"].Set(dt.Minute, rownum); break; } } } } private IIndex CreateIndex(string name, Type type) { if (type == typeof(NoIndexing)) return new NoIndex(); if (type == typeof(FullTextString)) return new FullTextIndex(_Path, name, false, true, _viewmanager._tokenizer); else if (type == typeof(string)) { byte len = Global.DefaultStringKeySize; if (_idxlen.TryGetValue(name, out len) == false) len = Global.DefaultStringKeySize; return new TypeIndexes(_Path, name, len); } else if (type == typeof(bool) || type == typeof(bool?)) return new BoolIndex(_Path, name, ".idx"); else if (type.IsEnum) return (IIndex)Activator.CreateInstance( typeof(EnumIndex<>).MakeGenericType(type), new object[] { _Path, name }); else return (IIndex)Activator.CreateInstance( typeof(TypeIndexes<>).MakeGenericType(type), new object[] { _Path, name, Global.DefaultStringKeySize }); } private void DeleteRowsWith(Guid guid) { // find bitmap for guid column MGRB gc = QueryColumnExpression(_docid, RDBExpression.Equal, guid); _deletedRows.InPlaceOR(gc); } private MGRB QueryColumnExpression(string colname, RDBExpression exp, object from) { int i = 0; if (_nocase.TryGetValue(colname, out i)) // no case query return _indexes[colname].Query(exp, ("" + from).ToLowerInvariant(), _viewData.Count()); else return _indexes[colname].Query(exp, from, _viewData.Count()); } private MGRB QueryColumnExpressionFromTo(string colname, object from, object to) { int i = 0; if (_nocase.TryGetValue(colname, out i)) // no case query return _indexes[colname].Query(("" + from).ToLowerInvariant(), ("" + to).ToLowerInvariant(), _viewData.Count()); else return _indexes[colname].Query(from, to, _viewData.Count()); } #endregion internal int Count(Expression> filter) { int totcount = 0; DateTime dt = FastDateTime.Now; if (filter == null) totcount = internalCount(); else { var ba = GenerateBitmap(filter); totcount = (int)ba.CountOnes(); } _log.Debug("Count items = " + totcount); _log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return totcount; } internal int Count(string filter) { int totcount = 0; DateTime dt = FastDateTime.Now; filter = filter.Trim(); if (filter == null || filter == "") totcount = internalCount(); else { _log.Debug("Count filter : " + filter); var ba = GenerateBitmap(filter); totcount = (int)ba.CountOnes(); } _log.Debug("Count items = " + totcount); _log.Debug("Count time (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return totcount; } private MGRB GenerateBitmap(Expression> filter) { MGRB ba = null; //new MGRB(); // check query cache //_queryCache.TryGetValue(filter, out ba);// FIX : showing incorrect results if (ba == null) { ba = new MGRB(); QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpressionFromTo); qv.Visit(filter); var delbits = _deletedRows.GetBits(); if (qv._bitmap.Count > 0) { MGRB qbits = (MGRB)qv._bitmap.Pop(); ba = qbits.AndNot(delbits); } else if (qv._stack.Count > 0) { var val = Convert.ToBoolean(qv._stack.Pop()); if (val == true) ba = new MGRB().Not(_viewData.Count()).AndNot(delbits); } //_queryCache.Add(filter, ba); } else { _log.Debug(" found in cache"); ba = ba.Copy(); } return ba; } private MGRB GenerateBitmap(string filter) { MGRB ba = null; //new MGRB(); // check query cache //_queryCache.TryGetValue(filter, out ba);// FIX : showing incorrect results if (ba == null) { ba = new MGRB(); LambdaExpression le = null; //if (_lambdacache.TryGetValue(filter, out le) == false) { le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null); // _lambdacache.Add(filter, le); } QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpressionFromTo); qv.Visit(le.Body); var delbits = _deletedRows.GetBits(); if (qv._bitmap.Count > 0) { MGRB qbits = (MGRB)qv._bitmap.Pop(); ba = qbits.AndNot(delbits); } else if (qv._stack.Count > 0) { var val = Convert.ToBoolean(qv._stack.Pop()); if (val == true) ba = new MGRB().Not(_viewData.Count()).AndNot(delbits); } //_queryCache.Add(filter, ba); } else { _log.Debug(" found in cache"); ba = ba.Copy(); } return ba; } private int internalCount() // note : don't use for Not() since _deletedRows are excluded { if (_rebuilding) while (_rebuilding) Thread.Sleep(10); // wait for rebuild to finish int c = _viewData.Count(); int cc = (int)_deletedRows.GetBits().CountOnes(); return c - cc; } internal Result QueryWithTypedResult(Expression> filter, int start, int count, string orderby) { DateTime dt = FastDateTime.Now; _log.Debug("query : " + _view.Name); var ba = GenerateBitmap(filter); List trows = null; if (_viewmanager.inTransaction()) { // query from transactions own data tran_data data = null; if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data)) { List rrows = new List(); foreach (var kv in data.rows) { foreach (var r in kv.Value) { object o = FastCreateObject(_view.Schema); rrows.Add((T)_rowfiller(o, r)); } } trows = rrows.FindAll(filter.Compile()); } } var order = SortBy(orderby); bool desc = false; if (orderby.ToLower().Contains(" desc")) desc = true; _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); dt = FastDateTime.Now; // exec query return rows return ReturnRowsTyped(ba, trows, start, count, order, desc); } internal Result QueryWithTypedResult(string filter, int start, int count, string orderby) { DateTime dt = FastDateTime.Now; _log.Debug("query : " + _view.Name); _log.Debug("query : " + filter); if (orderby != "") _log.Debug("order by : " + orderby); MGRB ba = new MGRB(); var delbits = _deletedRows.GetBits(); if (filter != "") ba = GenerateBitmap(filter); else ba = MGRB.Fill(_viewData.Count()).AndNot(delbits); var order = SortBy(orderby); bool desc = false; if (orderby.ToLower().Contains(" desc")) desc = true; _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); dt = FastDateTime.Now; // exec query return rows return ReturnRowsTyped(ba, null, start, count, order, desc); } private SafeDictionary> _sortcache = new SafeDictionary>(); internal List SortBy(string sortcol) { List sortlist = new List(); if (sortcol == "") return sortlist; string col = ""; foreach (var c in _schema.Columns) if (sortcol.ToLower().Contains(c.Key.ToLower())) { col = c.Key; break; } if (col == "") { _log.Debug("sort column not recognized : " + sortcol); return sortlist; } DateTime dt = FastDateTime.Now; if (_sortcache.TryGetValue(col, out sortlist) == false) { sortlist = new List(); int count = _viewData.Count(); IIndex idx = _indexes[col]; object[] keys = idx.GetKeys(); Array.Sort(keys); foreach (var k in keys) { var bi = idx.Query(RDBExpression.Equal, k, count).GetBitIndexes(); foreach (var i in bi) sortlist.Add(i); } _sortcache.Add(col, sortlist); } _log.Debug("Sort column = " + col + ", time (ms) = " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); return sortlist; } internal object GetAssembly(out string typename) { typename = _view.Schema.AssemblyQualifiedName; return File.ReadAllBytes(_view.Schema.Assembly.Location); } public ViewRowDefinition GetSchema() { return _schema; } int _lastrownumber = -1; object _rowlock = new object(); internal int NextRowNumber() { lock (_rowlock) { if (_lastrownumber == -1) _lastrownumber = internalCount(); return ++_lastrownumber; } } internal int ViewDelete(Expression> filter) { _log.Debug("delete : " + _view.Name); var ba = GenerateBitmap(filter); long count = ba.CountOnes(); if (count > 0) { _deletedRows.InPlaceOR(ba); _log.Debug("Deleted rows = " + count); InvalidateCaches(); if (_isDirty == false) WriteDirtyFile(); } return (int)count; } private object _dfile = new object(); private void WriteDirtyFile() { lock (_dfile) { _isDirty = true; if (File.Exists(_Path + _dirtyFilename) == false) File.WriteAllText(_Path + _dirtyFilename, "dirty"); } } internal int ViewDelete(string filter) { _log.Debug("delete : " + _view.Name); long count = 0; if (filter != "") { var ba = GenerateBitmap(filter); count = ba.CountOnes(); if (count > 0) { _deletedRows.InPlaceOR(ba); InvalidateCaches(); if (_isDirty == false) WriteDirtyFile(); } } return (int)count; } internal bool ViewInsert(Guid id, object row) { List l = new List(); l.Add(row); var r = ExtractRows(l); InsertRowsWithIndexUpdate(id, r); if (l.Count > 0) InvalidateCaches(); return true; } private void InvalidateCaches() { // invalidate sort cache _sortcache = new SafeDictionary>(); // inavidate query cache //_queryCache = new SafeDictionary(); // FIX : showing incorrect results } #region [ removed ] //SafeDictionary _lambdacache = new SafeDictionary(); //internal Result Query(string filter, int start, int count) //{ // return Query(filter, start, count, ""); //} //internal Result Query(Expression> filter, int start, int count) //{ // return Query(filter, start, count, ""); //} //internal Result Query(Expression> filter, int start, int count, string orderby) //{ // if (filter == null) // return Query(start, count); // DateTime dt = FastDateTime.Now; // _log.Debug("query : " + _view.Name); // MGRB ba = new MGRB(); // _queryCache.TryGetValue(filter, out ba); // if (ba == null) // { // ba = new MGRB(); // QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpFromTo); // qv.Visit(filter); // var delbits = _deletedRows.GetBits(); // if (qv._bitmap.Count > 0) // { // MGRB qbits = (MGRB)qv._bitmap.Pop(); // ba = qbits.AndNot(delbits); // } // else if (qv._stack.Count > 0) // { // var val = Convert.ToBoolean(qv._stack.Pop()); // if (val == true) // ba = new MGRB().Not(this.internalCount()).AndNot(delbits); // } // _queryCache.Add(filter, ba); // } // else // _log.Debug(" found in cache"); // List trows = null; // if (_viewmanager.inTransaction()) // { // // query from transaction own data // tran_data data = null; // if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data)) // { // List rrows = new List(); // foreach (var kv in data.rows) // { // foreach (var r in kv.Value) // { // object o = FastCreateObject(_view.Schema); // rrows.Add((T)_rowfiller(o, r)); // } // } // trows = rrows.FindAll(filter.Compile()); // } // } // var order = SortBy(orderby); // bool desc = false; // if (orderby.ToLower().Contains(" desc")) // desc = true; // _log.Debug("query bitmap done (ms) : " + FastDateTime.Now.Subtract(dt).TotalMilliseconds); // dt = FastDateTime.Now; // // exec query return rows // return ReturnRowsObject(ba, trows, start, count, order, desc); //} //internal Result Query2(Expression> filter, int start, int count) //{ // return Query2(filter, start, count, ""); //} //internal Result Query2(string filter, int start, int count) //{ // return Query2(filter, start, count, ""); //} #endregion } } ================================================ FILE: RaptorDB/Views/ViewManager.cs ================================================ using RaptorDB.Common; using System; using System.Collections.Generic; using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; namespace RaptorDB.Views { internal class ViewManager { public ViewManager(string viewfolder, IDocStorage objstore, IKeyStoreHF kvhf, ITokenizer tokenizer) { _Path = viewfolder; _objectStore = objstore; _kvhf = kvhf; _tokenizer = tokenizer; } private IKeyStoreHF _kvhf; private IDocStorage _objectStore; private ILog _log = LogManager.GetLogger(typeof(ViewManager)); private string _Path = ""; // list of views private SafeDictionary _views = new SafeDictionary(); // primary view list private SafeDictionary _primaryView = new SafeDictionary(); // like primary view list private SafeDictionary _otherViewTypes = new SafeDictionary(); // consistent views private SafeDictionary> _consistentViews = new SafeDictionary>(); // other views type->list of view names to call private SafeDictionary> _otherViews = new SafeDictionary>(); private TaskQueue _que = new TaskQueue(); private SafeDictionary _transactions = new SafeDictionary(); internal ITokenizer _tokenizer; internal int Count(string viewname, string filter) { ViewHandler view = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out view)) return view.Count(filter); _log.Error("view not found", viewname); return 0; } internal Result Query(string viewname, string filter, int start, int count) { return Query(viewname, filter, start, count, ""); } internal Result Query(string viewname, int start, int count) { ViewHandler view = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out view)) return view.Query(start, count); _log.Error("view not found", viewname); return new Result(false, new Exception("view not found : " + viewname)); } internal void Insert(string viewname, Guid docid, T data) { ViewHandler vman = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out vman)) { if (vman._view.isActive == false) { _log.Debug("view is not active, skipping insert : " + viewname); return; } if (vman._view.BackgroundIndexing) _que.AddTask(() => vman.Insert(docid, data)); else vman.Insert(docid, data); return; } _log.Error("view not found", viewname); } internal bool InsertTransaction(string viewname, Guid docid, T data) { ViewHandler vman = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out vman)) { if (vman._view.isActive == false) { _log.Debug("view is not active, skipping insert : " + viewname); return false; } return vman.InsertTransaction(docid, data); } _log.Error("view not found", viewname); return false; } internal object Fetch(Guid guid) { object b = null; _objectStore.GetObject(guid, out b); return b; } internal string GetPrimaryViewForType(Type type) { string vn = ""; if (type == null || type == typeof(object)) // reached the end return vn; // find direct if (_primaryView.TryGetValue(type, out vn)) return vn; // recurse basetype return GetPrimaryViewForType(type.BaseType); } internal List GetOtherViewsList(Type type) { List list = new List(); _otherViews.TryGetValue(type, out list); return list; } internal string GetViewName(Type type) // used for queries { string viewname = null; // find view from name viewname = GetPrimaryViewForType(type); if (viewname != "") return viewname; // search for viewtype here if (_otherViewTypes.TryGetValue(type, out viewname)) return viewname; return ""; } internal void RegisterView(View view) { view.Verify(); ViewHandler vh = null; if (_views.TryGetValue(view.Name.ToLower(), out vh)) { _log.Error("View already added and exists : " + view.Name); } else { vh = new ViewHandler(_Path, this); vh.SetView(view, _objectStore); _views.Add(view.Name.ToLower(), vh); _otherViewTypes.Add(view.GetType(), view.Name.ToLower()); // add view schema mapping _otherViewTypes.Add(view.Schema, view.Name.ToLower()); Type basetype = vh.GetFireOnType(); if (view.isPrimaryList) { _primaryView.Add(basetype, view.Name.ToLower()); } else { if (view.ConsistentSaveToThisView) AddToViewList(_consistentViews, basetype, view.Name); else AddToViewList(_otherViews, basetype, view.Name); } } } internal void ShutDown() { _log.Debug("View Manager shutdown"); _que.Shutdown(); List tasks = new List(); // shutdown views foreach (var v in _views) { try { tasks.Add(Task.Factory.StartNew(() => { _log.Debug(" shutting down view : " + v.Value._view.Name); v.Value.Shutdown(); })); } catch (Exception ex) { _log.Error(ex); } } Task.WaitAll(tasks.ToArray()); } internal List GetConsistentViews(Type type) { List list = new List(); _consistentViews.TryGetValue(type, out list); return list; } private void AddToViewList(SafeDictionary> diclist, Type fireontype, string viewname) { //foreach (var tn in view.FireOnTypes) { List list = null; Type t = fireontype;// Type.GetType(tn); if (diclist.TryGetValue(t, out list)) list.Add(viewname); else { list = new List(); list.Add(viewname); diclist.Add(t, list); } } } internal void Delete(Guid docid) { // remove from all views foreach (var v in _views) v.Value.Delete(docid); } internal void Rollback(int ID) { _log.Debug("ROLLBACK"); // rollback all views with tran id foreach (var v in _views) v.Value.RollBack(ID); _transactions.Remove(ID); } internal void Commit(int ID) { _log.Debug("COMMIT"); // commit all data in vews with tran id foreach (var v in _views) v.Value.Commit(ID); _transactions.Remove(ID); } internal bool isTransaction(string viewname) { return _views[viewname.ToLower()]._view.TransactionMode; } internal bool inTransaction() { bool b = false; return _transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out b); } internal void StartTransaction() { _transactions.Add(Thread.CurrentThread.ManagedThreadId, false); } internal Result Query(Expression> filter, int start, int count) { return Query(filter, start, count, ""); } internal Result Query(Expression> filter, int start, int count, string orderby) { string view = GetViewName(typeof(T)); ViewHandler vman = null; // find view from name if (_views.TryGetValue(view.ToLower(), out vman)) { return vman.QueryWithTypedResult(filter, start, count, orderby); } return new Result(false, new Exception("View not found")); } internal Result Query(string filter, int start, int count) { return Query(filter, start, count, ""); } internal Result Query(string filter, int start, int count, string orderby) { string view = GetViewName(typeof(T)); ViewHandler vman = null; // find view from name if (_views.TryGetValue(view.ToLower(), out vman)) { return vman.QueryWithTypedResult(filter, start, count, orderby); } return new Result(false, new Exception("View not found")); } internal int Count(Expression> filter) { string view = GetViewName(typeof(T)); ViewHandler vman = null; // find view from name if (_views.TryGetValue(view.ToLower(), out vman)) { return vman.Count(filter); } return 0; } internal void FreeMemory() { foreach (var v in _views) v.Value.FreeMemory(); } internal object GetAssemblyForView(string viewname, out string typename) { ViewHandler view = null; typename = ""; // find view from name if (_views.TryGetValue(viewname.ToLower(), out view)) { return view.GetAssembly(out typename); } return null; } internal List GetViews() { List o = new List(); foreach (var i in _views) o.Add(i.Value._view); return o; } internal ViewRowDefinition GetSchema(string view) { ViewHandler v = null; if (_views.TryGetValue(view.ToLower(), out v)) { return v.GetSchema(); } return null; } internal Result Query(string viewname, string filter, int start, int count, string orderby) { ViewHandler view = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out view)) return view.Query(filter, start, count, orderby); _log.Error("view not found", viewname); return new Result(false, new Exception("view not found : " + viewname)); } internal int ViewDelete(Expression> filter) { string view = GetViewName(typeof(T)); ViewHandler vman = null; // find view from name if (_views.TryGetValue(view.ToLower(), out vman)) { return vman.ViewDelete(filter); } return -1; } internal int ViewDelete(string viewname, string filter) { ViewHandler view = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out view)) return view.ViewDelete(filter); return -1; } internal bool ViewInsert(Guid id, T row) { string view = GetViewName(typeof(T)); ViewHandler vman = null; // find view from name if (_views.TryGetValue(view.ToLower(), out vman)) { return vman.ViewInsert(id, row); } return false; } internal bool ViewInsert(string viewname, Guid id, object row) { ViewHandler vman = null; // find view from name if (_views.TryGetValue(viewname.ToLower(), out vman)) { return vman.ViewInsert(id, row); } return false; } internal IKeyStoreHF GetKVHF() { return _kvhf; } } } ================================================ FILE: RaptorDB/Views/apimapper.cs ================================================ using System; using System.Collections.Generic; using System.Linq.Expressions; namespace RaptorDB.Views { internal class apimapper : IMapAPI { public apimapper(ViewManager man, ViewHandler vhandler) { _viewmanager = man; _viewhandler = vhandler; } ViewManager _viewmanager; ViewHandler _viewhandler; private ILog _log = LogManager.GetLogger(typeof(apimapper)); internal Dictionary> emit = new Dictionary>(); internal Dictionary> emitobj = new Dictionary>(); internal bool _RollBack = false; public void Log(string message) { _log.Debug(message); } public object Fetch(Guid guid) { return _viewmanager.Fetch(guid); } public void Emit(Guid docid, params object[] data) { if (data == null) return; List d = null; if (emit.Count == 0) { d = new List(); d.Add(data); emit.Add(docid, d); } else { if (emit.TryGetValue(docid, out d)) { d.Add(data); } else { d = new List(); d.Add(data); emit.Add(docid, d); } } } public void EmitObject(Guid docid, T doc) { if (doc == null) return; List d = null; if (emitobj.Count == 0) { d = new List(); d.Add(doc); emitobj.Add(docid, d); } else { if (emitobj.TryGetValue(docid, out d)) { d.Add(doc); } else { d = new List(); d.Add(doc); emitobj.Add(docid, d); } } } public void RollBack() { _RollBack = true; } public int Count(string viewname) { return _viewmanager.Count(viewname, ""); } public int Count(string ViewName, string Filter) { return _viewmanager.Count(ViewName, Filter); } public Result Query(Expression> Filter) { return _viewmanager.Query(Filter, 0, -1); } public Result Query(Expression> Filter, int start, int count) { return _viewmanager.Query(Filter, start, count); } public Result Query(string Filter) { return _viewmanager.Query(Filter, 0, -1); } public Result Query(string Filter, int start, int count) { return _viewmanager.Query(Filter, start, count); } public int Count(Expression> Filter) { return _viewmanager.Count(Filter); } public int NextRowNumber() { return _viewhandler.NextRowNumber(); } public Common.IKeyStoreHF GetKVHF() { return _viewmanager.GetKVHF(); } public T Fetch(Guid guid) where T : class { return (T)Fetch(guid); } } } ================================================ FILE: RaptorDB/WEB/bundle.css ================================================ .Container.svelte-obficw{width:100%;backface-visibility:hidden;will-change:overflow}.Left.svelte-obficw,.Middle.svelte-obficw{overflow:auto;height:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:none}.Left.svelte-obficw::-webkit-scrollbar,.Middle.svelte-obficw::-webkit-scrollbar{display:none}.Left.svelte-obficw{width:220px;float:left}.tab-links.svelte-obficw:after{display:block;clear:both;content:""}.tab-links.svelte-obficw li.svelte-obficw{margin:0px 1px;float:left;list-style:none}.tab-links.svelte-obficw{-webkit-padding-start:5px;border-radius:5px 5px 0px 0px;margin-bottom:0px;margin-top:10px}.tab-links.svelte-obficw div.svelte-obficw{padding:9px 15px;display:inline-block;border-radius:5px 5px 0px 0px;background:#333333;font-size:16px;font-weight:600;color:#999999;transition:all linear 0.15s;cursor:pointer}.tab-links.svelte-obficw div #close.svelte-obficw{margin-left:10px;padding:2px}.tab-links.svelte-obficw div #close.svelte-obficw:hover{background:red;color:white;text-align:center;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px;-moz-box-shadow:1px 1px 3px #000;-webkit-box-shadow:1px 1px 3px #000;box-shadow:1px 1px 3px #000}.tab-links.svelte-obficw div.svelte-obficw:hover{background:white;text-decoration:none}li.active.svelte-obficw div.svelte-obficw,li.active.svelte-obficw{background:white;border-radius:5px 5px 0px 0px;border:1px;color:#333}.closeall.svelte-obficw{margin-left:-35px;color:white;padding:8px;float:right !important;font-weight:bolder;text-align:center}.closeall.svelte-obficw:hover{background:red;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px} button.svelte-g32zaw,a.svelte-g32zaw{font:inherit;border:1px solid #cf0056;background:#cf0056;padding:0.5rem 1rem;color:white;border-radius:5px;box-shadow:1px 1px 3px rgba(0, 0, 0, 0.26);cursor:pointer;text-decoration:none}button.svelte-g32zaw:focus{outline:none}button.svelte-g32zaw:hover,button.svelte-g32zaw:active,a.svelte-g32zaw:hover,a.svelte-g32zaw:active{background:#e40763;border-color:#e40763;box-shadow:1px 1px 8px rgba(77, 51, 51, 0.26)}button.svelte-g32zaw:disabled,button.svelte-g32zaw:disabled:hover,button.svelte-g32zaw:disabled:active{background:#ccc;border-color:#ccc;color:#959595;box-shadow:none;cursor:not-allowed}.success.svelte-g32zaw{background:#01a129;border-color:#01a129}.success.svelte-g32zaw:hover,.success.svelte-g32zaw:active{background:#1ac745;border-color:#1ac745}.outline.svelte-g32zaw{background:transparent;color:#cf0056;box-shadow:none}.outline.svelte-g32zaw:hover,.outline.svelte-g32zaw:active{background:#ffc7de;box-shadow:none}.outline.svelte-g32zaw:disabled,.outline.svelte-g32zaw:disabled:hover,.outline.svelte-g32zaw:disabled:active{background:transparent;color:#ccc}.outline.success.svelte-g32zaw{border-color:#01a129;color:#01a129}.outline.success.svelte-g32zaw:hover,.outline.success.svelte-g32zaw:active{background:#c2ffd1} button.accordion.svelte-k0merd{background-color:#333333;color:#bbb;cursor:pointer;padding:18px;width:97%;border:none;text-align:left;outline:none;font-size:15px;transition:0.3s;margin-bottom:5px}button.accordion.active.svelte-k0merd{background-color:#666666;color:#eee;margin-bottom:0}button.accordion.svelte-k0merd:hover{background-color:#999999}button.accordion.svelte-k0merd:after{content:"\02795";font-size:13px;color:#666666;float:right;margin-left:5px}button.accordion.active.svelte-k0merd:after{content:"\2796"}div.panel.svelte-k0merd{padding:0 18px;background-color:white;max-height:0;overflow:hidden;transition:0.1s ease-out;opacity:0}div.panel.show.svelte-k0merd{opacity:1;max-height:500px;min-height:100px;padding:5px;width:203px}div.panel.show.svelte-k0merd label.svelte-k0merd{cursor:pointer;display:block;padding:5px}div.panel.show.svelte-k0merd label.svelte-k0merd:hover{background-color:#999999} .modal-backdrop.svelte-f1bfsy{position:fixed;top:0;left:0;width:100%;height:100vh;background:rgba(0, 0, 0, 0.55);z-index:10}.modal.svelte-f1bfsy{position:absolute;top:10vh;width:80%;max-height:80vh;background:white;border-radius:5px;z-index:100;box-shadow:0 2px 8px rgba(0, 0, 0, 0.96)}h1.svelte-f1bfsy{padding:1rem;margin:0;border-bottom:1px solid #ccc;font-family:"Roboto Slab", sans-serif}.content.svelte-f1bfsy{padding:1rem}footer.svelte-f1bfsy{padding:1rem}@media(min-width: 768px){.modal.svelte-f1bfsy{width:40rem;left:calc(50% - 20rem)}} .link.svelte-1bl9b36{color:blue;text-decoration:underline;cursor:pointer}.ListBox.svelte-1bl9b36{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-1bl9b36 label.svelte-1bl9b36{color:blue;text-decoration:underline;display:block;padding:2px}.ListBox.svelte-1bl9b36 label.svelte-1bl9b36:hover{background-color:#c0c0c0} .link.svelte-m3ntri{color:blue;text-decoration:underline;cursor:pointer;display:block}.link2.svelte-m3ntri{color:blue;text-decoration:underline;cursor:pointer;margin-left:20px}.ListBox.svelte-m3ntri{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll} .bold.svelte-yogi1w{font-weight:700}button.svelte-yogi1w{margin-right:20px} .link.svelte-k8kdho{color:blue;text-decoration:underline;cursor:pointer;margin-left:20px} .link.svelte-6usscg{color:blue;text-decoration:underline;cursor:pointer;display:block}.ListBox.svelte-6usscg{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-6usscg label.svelte-6usscg{color:blue;text-decoration:underline;display:block;padding:2px}.ListBox.svelte-6usscg label.svelte-6usscg:hover{background-color:#c0c0c0} .link.svelte-15b96z{color:blue;text-decoration:underline;cursor:pointer}.ListBox.svelte-15b96z{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-15b96z label.svelte-15b96z{display:block;padding:2px}.ListBox.svelte-15b96z label.svelte-15b96z:hover{background-color:#c0c0c0} .bold.svelte-mdntjp{font-weight:700}.ListBox.svelte-mdntjp{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll;height:150px}.collist.svelte-mdntjp{display:block;font-weight:bold}.schema_def.svelte-mdntjp th.svelte-mdntjp{background-color:#cccccc}.schema_def.svelte-mdntjp td.svelte-mdntjp:nth-child(odd){font-weight:bold;text-align:right}.schema_def.svelte-mdntjp td.svelte-mdntjp:nth-child(even){color:green} .link.svelte-1qrhxsw{color:blue;text-decoration:underline;cursor:pointer}.cssTbl.svelte-1qrhxsw{margin:0px;padding:3px;font-family:Arial;font-weight:normal}.cssTbl.svelte-1qrhxsw table.svelte-1qrhxsw{border-spacing:0;width:100%;margin:0px;padding:0px}.cssTbl.svelte-1qrhxsw th.svelte-1qrhxsw{background-color:#333333;padding:5px;cursor:pointer;font-weight:normal;color:white}.cssTbl.svelte-1qrhxsw th.svelte-1qrhxsw:hover{color:red}.cssTbl.svelte-1qrhxsw tr.svelte-1qrhxsw:nth-child(odd){background-color:#c0c0c0}.cssTbl.svelte-1qrhxsw tr.svelte-1qrhxsw:nth-child(even){background-color:#ffffff}.cssTbl.svelte-1qrhxsw td.svelte-1qrhxsw{vertical-align:middle;border:1px solid #666666;text-align:left;padding:5px;color:#000000} ================================================ FILE: RaptorDB/WEB/bundle.js ================================================ var app=function(){"use strict";function t(){}const e=t=>t;function n(t,e){for(const n in e)t[n]=e[n];return t}function o(t){return t()}function c(){return Object.create(null)}function l(t){t.forEach(o)}function a(t){return"function"==typeof t}function i(t,e){return t!=t?e==e:t!==e||t&&"object"==typeof t||"function"==typeof t}function r(t,e,n){if(t){const o=s(t,e,n);return t[0](o)}}function s(t,e,o){return t[1]?n({},n(e.$$scope.ctx,t[1](o?o(e):{}))):e.$$scope.ctx}function d(t,e,o,c){return t[1]?n({},n(e.$$scope.changed||{},t[1](c?c(o):{}))):e.$$scope.changed||{}}let u="undefined"!=typeof window?()=>window.performance.now():()=>Date.now(),v=t=>requestAnimationFrame(t);const f=new Set;let p,h=!1;function m(){f.forEach(t=>{t[0](u())||(f.delete(t),t[1]())}),(h=f.size>0)&&v(m)}function g(t,e){t.appendChild(e)}function b(t,e,n){t.insertBefore(e,n||null)}function w(t){t.parentNode.removeChild(t)}function $(t,e){for(let n=0;nt.removeEventListener(e,n,o)}function S(t,e,n){null==n?t.removeAttribute(e):t.setAttribute(e,n)}function j(t,e){e=""+e,t.data!==e&&(t.data=e)}function O(t,e,n){t.style.setProperty(e,n)}function D(t,e){for(let n=0;n>>0}(d)}_${i}`;if(!M[u]){if(!p){const t=x("style");document.head.appendChild(t),p=t.sheet}M[u]=!0,p.insertRule(`@keyframes ${u} ${d}`,p.cssRules.length)}const v=t.style.animation||"";return t.style.animation=`${v?`${v}, `:""}${u} ${o}ms linear ${c}ms 1 both`,T+=1,u}function E(t,e){t.style.animation=(t.style.animation||"").split(", ").filter(e?t=>t.indexOf(e)<0:t=>-1===t.indexOf("__svelte")).join(", "),e&&!--T&&v(()=>{if(T)return;let t=p.cssRules.length;for(;t--;)p.deleteRule(t);M={}})}function q(t){I=t}function R(t){(function(){if(!I)throw new Error("Function called outside component initialization");return I})().$$.on_mount.push(t)}function F(){const t=I;return(e,n)=>{const o=t.$$.callbacks[e];if(o){const c=V(e,n);o.slice().forEach(e=>{e.call(t,c)})}}}function A(t,e){const n=t.$$.callbacks[e.type];n&&n.slice().forEach(t=>t(e))}const G=[],H=[],z=[],U=[],J=Promise.resolve();let P,K=!1;function X(){K||(K=!0,J.then(Z))}function Q(){return X(),J}function W(t){z.push(t)}function Y(t){U.push(t)}function Z(){const t=new Set;do{for(;G.length;){const t=G.shift();q(t),tt(t.$$)}for(;H.length;)H.pop()();for(let e=0;e{nt.delete(t),o&&(n&&t.d(1),o())}),t.o(e)}}function rt(n,o,c,i){let r=o(n,c),s=i?0:1,d=null,p=null,g=null;function b(){g&&E(n,g)}function w(t,e){const n=t.b-s;return e*=Math.abs(n),{a:s,b:t.b,d:n,duration:e,start:t.start,end:t.start+e,group:t.group}}function $(o){const{delay:c=0,duration:a=300,easing:i=e,tick:$=t,css:x}=r,y={start:u()+c,b:o};o||(y.group=ot,ot.r+=1),d?p=y:(x&&(b(),g=B(n,s,o,a,c,i,x)),o&&$(0,1),d=w(y,a),W(()=>et(n,o,"start")),function(t){let e;h||(h=!0,v(m)),new Promise(n=>{f.add(e=[t,n])})}(t=>{if(p&&t>p.start&&(d=w(p,a),p=null,et(n,d.b,"start"),x&&(b(),g=B(n,s,d.b,d.duration,0,i,r.css))),d)if(t>=d.end)$(s=d.b,1-s),et(n,d.b,"end"),p||(d.b?b():--d.group.r||l(d.group.c)),d=null;else if(t>=d.start){const e=t-d.start;s=d.a+d.d*i(e/d.duration),$(s,1-s)}return!(!d&&!p)}))}return{run(t){a(r)?(P||(P=Promise.resolve()).then(()=>{P=null}),P).then(()=>{r=r(),$(t)}):$(t)},end(){b(),d=p=null}}}function st(t,e){t.d(1),e.delete(t.key)}function dt(t,e,n,o,c,l,a,i,r,s,d,u){let v=t.length,f=l.length,p=v;const h={};for(;p--;)h[t[p].key]=p;const m=[],g=new Map,b=new Map;for(p=f;p--;){const t=u(c,l,p),i=n(t);let r=a.get(i);r?o&&r.p(e,t):(r=s(i,t)).c(),g.set(i,m[p]=r),i in h&&b.set(i,Math.abs(p-h[i]))}const w=new Set,$=new Set;function x(t){at(t,1),t.m(i,d),a.set(t.key,t),d=t.first,f--}for(;v&&f;){const e=m[f-1],n=t[v-1],o=e.key,c=n.key;e===n?(d=e.first,v--,f--):g.has(c)?!a.has(o)||w.has(o)?x(e):$.has(c)?v--:b.get(o)>b.get(c)?($.add(o),x(e)):(w.add(c),v--):(r(n,a),v--)}for(;v--;){const e=t[v];g.has(e.key)||r(e,a)}for(;f;)x(m[f-1]);return m}function ut(t,e,n){-1!==t.$$.props.indexOf(e)&&(t.$$.bound[e]=n,n(t.$$.ctx[e]))}function vt(t,e,n){const{fragment:c,on_mount:i,on_destroy:r,after_update:s}=t.$$;c.m(e,n),W(()=>{const e=i.map(o).filter(a);r?r.push(...e):l(e),t.$$.on_mount=[]}),s.forEach(W)}function ft(t,e){t.$$.fragment&&(l(t.$$.on_destroy),t.$$.fragment.d(e),t.$$.on_destroy=t.$$.fragment=null,t.$$.ctx={})}function pt(e,n,o,a,i,r){const s=I;q(e);const d=n.props||{},u=e.$$={fragment:null,ctx:null,props:r,update:t,not_equal:i,bound:c(),on_mount:[],on_destroy:[],before_update:[],after_update:[],context:new Map(s?s.$$.context:[]),callbacks:c(),dirty:null};let v=!1;var f;u.ctx=o?o(e,d,(t,n)=>{u.ctx&&i(u.ctx[t],u.ctx[t]=n)&&(u.bound[t]&&u.bound[t](n),v&&function(t,e){t.$$.dirty||(G.push(t),X(),t.$$.dirty=c()),t.$$.dirty[e]=!0}(e,t))}):d,u.update(),v=!0,l(u.before_update),u.fragment=a(u.ctx),n.target&&(n.hydrate?u.fragment.l((f=n.target,Array.from(f.childNodes))):u.fragment.c(),n.intro&&at(e.$$.fragment),vt(e,n.target,n.anchor),Z()),q(s)}class ht{$destroy(){ft(this,1),this.$destroy=t}$on(t,e){const n=this.$$.callbacks[t]||(this.$$.callbacks[t]=[]);return n.push(e),()=>{const t=n.indexOf(e);-1!==t&&n.splice(t,1)}}$set(){}}function mt(t,e,n){const o=Object.create(t);return o.panel=e[n],o}function gt(t,e,n){const o=Object.create(t);return o.name=e[n],o}function bt(t){var e,n,o,c=t.panel.description;function l(){return t.click_handler_1(t)}return{c(){e=x("label"),n=y(c),S(e,"id",t.panel.name),S(e,"class","svelte-k0merd"),o=C(e,"click",l)},m(t,o){b(t,e,o),g(e,n)},p(e,n){t=n},d(t){t&&w(e),o()}}}function wt(t){var e,n,o,c,l,a,i=t.name;function r(){return t.click_handler(t)}for(var s=t.navdata[t.name],d=[],u=0;uWelcome to the RaptorDB Web Studio\n\t\t\t You can use this Web interface to query and see configuration information.\n\t\t\t ",S(e,"class","tab-content"),N(e,"active",t.active)},m(t,n){b(t,e,n)},p(t,n){t.active&&N(e,"active",n.active)},d(t){t&&w(e)}}}function _t(e){var n,o=e.active&&kt(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=kt(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function Ct(t,e,n){let{active:o=!1}=e;return t.$set=(t=>{"active"in t&&n("active",o=t.active)}),{active:o}}class St extends ht{constructor(t){super(),pt(this,t,Ct,_t,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function jt(t,e,n){const o=Object.create(t);return o.r=e[n],o}function Ot(t,e,n){const o=Object.create(t);return o.row=e[n],o}function Dt(t,e,n){const o=Object.create(t);return o.col=e[n],o}function Lt(e){var n;return{c(){(n=x("p")).textContent="No data"},m(t,e){b(t,n,e)},p:t,d(t){t&&w(n)}}}function Nt(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,_,D,L,N,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt=t.rows.length,et=t.totalrows.toLocaleString(),nt=t.totalpages.toLocaleString(),ot=[],ct=new Map,lt=t.totalpages.toLocaleString(),at=Object.keys(t.rows[0]);const it=t=>t.col;for(var rt=0;rt",_=k(),(D=x("label")).textContent="Rows per Page :",L=k(),(N=x("label")).textContent="10",V=k(),(I=x("label")).textContent="25",T=k(),(M=x("label")).textContent="50",B=k(),(E=x("label")).textContent="100",q=k(),R=x("div"),F=x("table"),A=x("tr"),l=0;l",S(s,"class","link svelte-1qrhxsw"),S(m,"class","link svelte-1qrhxsw"),O(D,"padding-left","100px"),S(N,"class","link svelte-1qrhxsw"),O(N,"padding-left","10px"),S(I,"class","link svelte-1qrhxsw"),O(I,"padding-left","10px"),S(M,"class","link svelte-1qrhxsw"),O(M,"padding-left","10px"),S(E,"class","link svelte-1qrhxsw"),O(E,"padding-left","10px"),S(r,"class","pager"),S(A,"class","svelte-1qrhxsw"),S(F,"class","svelte-1qrhxsw"),S(R,"class","cssTbl svelte-1qrhxsw"),S(U,"class","link svelte-1qrhxsw"),S(Y,"class","link svelte-1qrhxsw"),S(z,"class","pager"),Z=[C(s,"click",t.prevpage),C(m,"click",t.nextpage),C(N,"click",t.page10),C(I,"click",t.page25),C(M,"click",t.page50),C(E,"click",t.page100),C(U,"click",t.prevpage),C(Y,"click",t.nextpage)]},m(t,l){for(b(t,e,l),g(e,n),g(e,o),g(e,c),g(e,a),b(t,i,l),b(t,r,l),g(r,s),g(r,d),g(r,u),g(u,v),g(u,f),g(u,p),g(r,h),g(r,m),g(r,_),g(r,D),g(r,L),g(r,N),g(r,V),g(r,I),g(r,T),g(r,M),g(r,B),g(r,E),b(t,q,l),b(t,R,l),g(R,F),g(F,A),w=0;wt.r;for(var i=0;i0?Nt:Lt}var c=o(e),l=c(e);return{c(){l.c(),n=_()},m(t,e){l.m(t,e),b(t,n,e)},p(t,e){c===(c=o(e))&&l?l.p(t,e):(l.d(1),(l=c(e))&&(l.c(),l.m(n.parentNode,n)))},i:t,o:t,d(t){l.d(t),t&&w(n)}}}function qt(t,e,n){let{rows:o=[],totalrows:c=1,page:l=1}=e;const a=F();let i,r=10,s="",d="";function u(){a("refresh",{start:(l-1)*r,count:r,sort:s+" "+d,page:l})}function v(t){s!==t?(s=t,d="ASC"):d="ASC"===d?"DESC":"ASC",u()}function f(t){a("showdoc",t)}return t.$set=(t=>{"rows"in t&&n("rows",o=t.rows),"totalrows"in t&&n("totalrows",c=t.totalrows),"page"in t&&n("page",l=t.page)}),t.$$.update=((t={totalrows:1,rowspp:1})=>{(t.totalrows||t.rowspp)&&n("totalpages",i=Math.ceil(c/r))}),{rows:o,totalrows:c,page:l,nextpage:function(){l1&&n("page",l-=1),u()},sortby:v,showdoc:f,page10:function(){n("rowspp",r=10),n("page",l=1),u()},page25:function(){n("rowspp",r=25),n("page",l=1),u()},page50:function(){n("rowspp",r=50),n("page",l=1),u()},page100:function(){n("rowspp",r=100),n("page",l=1),u()},totalpages:i,click_handler:function({col:t}){return v(t)},click_handler_1:function({row:t,r:e}){return f(t[e])}}}class Rt extends ht{constructor(t){super(),pt(this,t,qt,Et,i,["rows","totalrows","page"])}}function Ft(t,e,n){const o=Object.create(t);return o.col=e[n],o}function At(t,e,n){const o=Object.create(t);return o.vn=e[n],o}function Gt(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,y,_,j,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,Y,Z,tt,et,nt,ot,rt,ut,vt=[],ft=new Map,pt=[],ht=new Map,mt=t.viewnames;const gt=t=>t.vn;for(var bt=0;btt.col;for(bt=0;btt.select_change_handler.call(c)),O(c,"width","200px"),O(c,"font-size","larger"),S(u,"align","top"),S(m,"id","columns"),O(m,"height","150px"),O(m,"width","150px"),O(m,"display","inline-block"),S(m,"class","ListBox svelte-m3ntri"),S(_,"class","link2 svelte-m3ntri"),S(L,"class","link2 svelte-m3ntri"),S(I,"class","link2 svelte-m3ntri"),S(M,"class","link2 svelte-m3ntri"),S(E,"class","link2 svelte-m3ntri"),S(R,"class","link2 svelte-m3ntri"),S(A,"class","link2 svelte-m3ntri"),S(H,"class","link2 svelte-m3ntri"),O(H,"float","right"),S(P,"id","filter"),S(P,"cols","80"),O(P,"height","156px"),O(P,"resize","none"),O(P,"border-color","grey"),O(P,"border-width","medium"),O(X,"width","200px"),O(X,"height","40px"),O(Y,"width","200px"),O(Y,"height","40px"),S(e,"class","tab-content"),N(e,"active",t.active),ut=[C(c,"change",t.select_change_handler),C(c,"change",t.schemachanged),C(_,"click",t.click_handler_1),C(L,"click",t.click_handler_2),C(I,"click",t.click_handler_3),C(M,"click",t.click_handler_4),C(E,"click",t.click_handler_5),C(R,"click",t.click_handler_6),C(A,"click",t.click_handler_7),C(H,"click",t.click_handler_8),C(P,"input",t.textarea_input_handler),C(X,"click",t.click_handler_9),C(Y,"click",t.excel)]},m(l,w){for(b(l,e,w),g(e,n),g(e,o),g(e,c),bt=0;bt{yt[a]=null}),lt(),(ot=yt[nt])||(ot=yt[nt]=xt[nt](n)).c(),at(ot,1),ot.m(e,null)),t.active&&N(e,"active",n.active)},i(t){rt||(at(ot),rt=!0)},o(t){it(ot),rt=!1},d(t){for(t&&w(e),bt=0;btn=!1)}function r(e){t.datatable_totalrows_binding.call(null,e),o=!0,Y(()=>o=!1)}function s(e){t.datatable_page_binding.call(null,e),c=!0,Y(()=>c=!1)}let d={};void 0!==t.data&&(d.rows=t.data),void 0!==t.totalrows&&(d.totalrows=t.totalrows),void 0!==t.page&&(d.page=t.page);var u=new Rt({props:d});return H.push(()=>ut(u,"rows",i)),H.push(()=>ut(u,"totalrows",r)),H.push(()=>ut(u,"page",s)),u.$on("refresh",t.refresh_handler),u.$on("showdoc",t.showdoc_handler),{c(){a&&a.c(),e=k(),u.$$.fragment.c()},m(t,n){a&&a.m(t,n),b(t,e,n),vt(u,t,n),l=!0},p(t,l){null!=l.data?a?a.p(t,l):((a=Pt(l)).c(),a.m(e.parentNode,e)):a&&(a.d(1),a=null);var i={};!n&&t.data&&(i.rows=l.data),!o&&t.totalrows&&(i.totalrows=l.totalrows),!c&&t.page&&(i.page=l.page),u.$set(i)},i(t){l||(at(u.$$.fragment,t),l=!0)},o(t){it(u.$$.fragment,t),l=!1},d(t){a&&a.d(t),t&&w(e),ft(u,t)}}}function Pt(t){var e,n,o;return{c(){e=x("pre"),n=y("Query time (+render) : "),o=y(t.qtime)},m(t,c){b(t,e,c),g(e,n),g(e,o)},p(t,e){t.qtime&&j(o,e.qtime)},d(t){t&&w(e)}}}function Kt(t){var e,n,o=t.active&&Gt(t);return{c(){o&&o.c(),e=_()},m(t,c){o&&o.m(t,c),b(t,e,c),n=!0},p(t,n){n.active?o?(o.p(t,n),at(o,1)):((o=Gt(n)).c(),at(o,1),o.m(e.parentNode,e)):o&&(ct(),it(o,1,1,()=>{o=null}),lt())},i(t){n||(at(o),n=!0)},o(t){it(o),n=!1},d(t){o&&o.d(t),t&&w(e)}}}function Xt(t,e,n){let{active:o=!1}=e,c="",l=[],a=[],i=null,r=0,s=10,d="",u="",v=1,f=1,p="",h="";function m(){n("page",f=1),n("start",r=0),window.GET("RaptorDB/GetSchema?view="+c,function(t){var e=[];t.Rows.forEach(t=>e.push(t.ColumnName)),n("colnames",a=e)})}function g(){n("errormsg",p="");var t="raptordb/views/"+c+(""!==d?"?"+d:"")+"?start="+r+"?count="+s+(" "!==u?"?orderby="+u:""),e=(new Date).getTime();window.GET(t,function(t){n("totalrows",v=t.TotalCount),n("data",i=t.Rows),n("qtime",h=(new Date).getTime()-e+"ms")},function(t){n("errormsg",p=t),n("data",i=[])})}function b(t){n("start",r=t.detail.start),s=t.detail.count,u=t.detail.sort,g()}function w(t){n("filter",d+=t)}return R(()=>{n("errormsg",p=""),window.GET("RaptorDB/GetViews",function(t){var e=[];t.Rows.forEach(t=>e.push(t.Name)),n("viewnames",l=e),e.length>0&&(n("viewname",c=e[0]),m())})}),t.$set=(t=>{"active"in t&&n("active",o=t.active)}),{active:o,viewname:c,viewnames:l,colnames:a,data:i,start:r,filter:d,totalrows:v,page:f,errormsg:p,qtime:h,schemachanged:m,run:g,excel:function(){var t=window.ServerURL+"RaptorDB/ExcelExport/"+c+"?"+d;window.open(t)},refresh:b,addcolumn:w,showdoc_handler:function(e){A(t,e)},select_change_handler:function(){c=L(this),n("viewname",c),n("viewnames",l)},click_handler:function({col:t}){return w(" "+t)},click_handler_1:function(){return w(" AND")},click_handler_2:function(){return w(" OR")},click_handler_3:function(){return w(".between(,)")},click_handler_4:function(){return w(".in(,)")},click_handler_5:function(){return w(".year")},click_handler_6:function(){return w(".month")},click_handler_7:function(){return w(".day")},click_handler_8:function(){const t=d="";return n("filter",d),t},textarea_input_handler:function(){d=this.value,n("filter",d)},click_handler_9:function(){n("page",f=1),n("start",r=0),g()},datatable_rows_binding:function(t){n("data",i=t)},datatable_totalrows_binding:function(t){n("totalrows",v=t)},datatable_page_binding:function(t){n("page",f=t)},refresh_handler:function(t){return b(t)}}}class Qt extends ht{constructor(t){super(),pt(this,t,Xt,Kt,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Wt(t,e,n){const o=Object.create(t);return o.l=e[n],o}function Yt(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p,h,m,$,_,O,D,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt,ut,vt,ft,pt,ht,mt,gt,bt,wt,$t,xt,yt,kt,_t,Ct,St,jt,Ot,Dt,Lt,Nt,Vt,It,Tt=t.DataFolderSize.toLocaleString()+" bytes",Mt=t.DocumentCount.toLocaleString(),Bt=t.FileCount.toLocaleString(),Et=t.HighFrequncyItems.toLocaleString(),qt=[],Rt=new Map,Ft=t.logs;const At=t=>t.l;for(var Gt=0;Gt{p()}),t.$set=(t=>{"active"in t&&n("active",o=t.active)}),{active:o,DataFolderSize:c,DocumentCount:l,FileCount:a,HighFrequncyItems:i,MemoryUsage:r,NumberOfViews:s,OSVersion:d,RaptorDBVersion:u,Uptime:v,logs:f,refresh:p}}class ne extends ht{constructor(t){super(),pt(this,t,ee,te,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function oe(t,e,n){const o=Object.create(t);return o.v=e[n],o}function ce(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,V=t.total>0&&le(t);return{c(){e=x("div"),(n=x("label")).textContent="Search:",o=k(),c=x("input"),a=k(),(i=x("button")).textContent="Get",r=k(),s=x("br"),d=k(),u=x("br"),v=k(),(f=x("label")).textContent="Documents found :",p=k(),h=x("label"),m=y(t.total),$=k(),_=x("label"),D=k(),V&&V.c(),S(n,"align","top"),S(c,"id","search"),c.autofocus=!0,O(c,"width","400px"),O(i,"width","50px"),S(f,"align","top"),O(h,"font-weight","700"),O(_,"color","red"),S(_,"id","err"),S(e,"class","tab-content"),N(e,"active",t.active),L=[C(c,"input",t.input_input_handler),C(c,"keypress",t.keypress_handler),C(i,"click",t.docsearch)]},m(l,w){b(l,e,w),g(e,n),g(e,o),g(e,c),c.value=t.find,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(e,f),g(e,p),g(e,h),g(h,m),g(e,$),g(e,_),g(e,D),V&&V.m(e,null),c.focus()},p(t,n){t.find&&c.value!==n.find&&(c.value=n.find),t.total&&j(m,n.total),n.total>0?V?V.p(t,n):((V=le(n)).c(),V.m(e,null)):V&&(V.d(1),V=null),t.active&&N(e,"active",n.active)},d(t){t&&w(e),V&&V.d(),l(L)}}}function le(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,N,V,I,T,M,B=[],E=new Map,q=t.versions;const R=t=>t.v;for(var F=0;F",p=k(),(h=x("td")).innerHTML='
',m=k(),$=x("tr"),_=x("td"),D=x("div"),F=0;F{"active"in t&&n("active",c=t.active)}),t.$$.update=((t={total:1,count:1})=>{(t.total||t.count)&&n("pages",o=Math.ceil(s/re))}),{active:c,find:l,page:a,versions:i,jsondata:r,total:s,docsearch:u,next:function(){a1&&n("page",a-=1),u()},showver:v,pages:o,input_input_handler:function(){l=this.value,n("find",l)},keypress_handler:function(t){13==(t.keyCode||t.which)&&u()},click_handler:function({v:t}){return v(t)}}}class de extends ht{constructor(t){super(),pt(this,t,se,ie,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function ue(t){var e,n,o,c,a,i,r,s,d,u,v,f,p=""!==t.doc&&ve(t);return{c(){e=x("div"),(n=x("label")).textContent="docid:",o=k(),c=x("input"),a=k(),(i=x("button")).textContent="Get",r=k(),s=x("br"),d=k(),u=x("br"),v=k(),p&&p.c(),S(n,"align","top"),S(c,"id","docid"),c.autofocus=!0,O(c,"width","400px"),O(i,"width","50px"),S(e,"class","tab-content"),N(e,"active",t.active),f=[C(c,"input",t.input_input_handler),C(c,"keypress",t.keypress_handler),C(i,"click",t.docview)]},m(l,f){b(l,e,f),g(e,n),g(e,o),g(e,c),c.value=t.docid,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),p&&p.m(e,null),c.focus()},p(t,n){t.docid&&c.value!==n.docid&&(c.value=n.docid),""!==n.doc?p?p.p(t,n):((p=ve(n)).c(),p.m(e,null)):p&&(p.d(1),p=null),t.active&&N(e,"active",n.active)},d(t){t&&w(e),p&&p.d(),l(f)}}}function ve(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p,h;return{c(){e=x("div"),(n=x("label")).textContent="Document Revisions :",o=k(),c=x("label"),l=k(),(a=x("label")).textContent="See Revisions",i=k(),r=x("br"),s=k(),(d=x("label")).textContent="Document JSON :",u=k(),v=x("div"),f=x("div"),p=y(t.doc),S(n,"align","top"),O(c,"font-weight","700"),S(a,"class","link svelte-k8kdho"),O(a,"margin-left","5px"),S(f,"class","JSON"),S(f,"id","data"),S(v,"class","JSONArea"),S(e,"class","disp"),h=C(a,"click",t.showrevs)},m(t,h){b(t,e,h),g(e,n),g(e,o),g(e,c),g(e,l),g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(v,f),g(f,p)},p(t,e){t.doc&&j(p,e.doc)},d(t){t&&w(e),h()}}}function fe(e){var n,o=e.active&&ue(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=ue(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function pe(t,e,n){let{active:o=!1,docid:c=null}=e,l="";const a=F();function i(){n("doc",l=""),window.LOAD("/raptordb/docget?"+c,function(t){n("doc",l=t)})}return R(()=>{null!=c&&i()}),t.$set=(t=>{"active"in t&&n("active",o=t.active),"docid"in t&&n("docid",c=t.docid)}),{active:o,docid:c,doc:l,docview:i,showrevs:function(){a("showrevs",c)},input_input_handler:function(){c=this.value,n("docid",c)},keypress_handler:function(t){13==(t.keyCode||t.which)&&i()}}}class he extends ht{constructor(t){super(),pt(this,t,pe,fe,i,["active","docid"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}get docid(){return this.$$.ctx.docid}set docid(t){this.$set({docid:t}),Z()}}function me(t,e,n){const o=Object.create(t);return o.v=e[n],o}function ge(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D=t.versions.length,L=t.versions.length>0&&be(t);return{c(){e=x("div"),(n=x("label")).textContent="docid:",o=k(),c=x("input"),a=k(),(i=x("button")).textContent="Get",r=k(),s=x("br"),d=k(),u=x("br"),v=k(),(f=x("label")).textContent="Document Revisions :",p=k(),h=x("label"),m=y(D),$=k(),L&&L.c(),S(n,"align","top"),S(c,"id","docid"),c.autofocus=!0,O(c,"width","400px"),O(i,"width","50px"),S(f,"align","top"),O(h,"font-weight","700"),S(e,"class","tab-content"),N(e,"active",t.active),_=[C(c,"input",t.input_input_handler),C(c,"keypress",t.fillhistory),C(i,"keypress",t.keypress_handler)]},m(l,w){b(l,e,w),g(e,n),g(e,o),g(e,c),c.value=t.docid,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(e,f),g(e,p),g(e,h),g(h,m),g(e,$),L&&L.m(e,null),c.focus()},p(t,n){t.docid&&c.value!==n.docid&&(c.value=n.docid),t.versions&&D!==(D=n.versions.length)&&j(m,D),n.versions.length>0?L?L.p(t,n):((L=be(n)).c(),L.m(e,null)):L&&(L.d(1),L=null),t.active&&N(e,"active",n.active)},d(t){t&&w(e),L&&L.d(),l(_)}}}function be(t){var e,n,o,c,l,a,i,r,s,d=[],u=new Map,v=t.versions;const f=t=>t.v;for(var p=0;p{null!=c&&i()}),t.$set=(t=>{"active"in t&&n("active",o=t.active),"docid"in t&&n("docid",c=t.docid)}),{active:o,docid:c,jsondata:l,versions:a,fillhistory:i,showver:r,input_input_handler:function(){c=this.value,n("docid",c)},keypress_handler:function(t){13==(t.keyCode||t.which)&&i()},click_handler:function({v:t}){return r(t.Version)}}}class ye extends ht{constructor(t){super(),pt(this,t,xe,$e,i,["active","docid"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}get docid(){return this.$$.ctx.docid}set docid(t){this.$set({docid:t}),Z()}}function ke(t,e,n){const o=Object.create(t);return o.v=e[n],o}function _e(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt=[],ut=new Map,vt=t.keys;const ft=t=>t.v;for(var pt=0;pt",Q=k(),(W=x("td")).innerHTML='
',Y=k(),Z=x("tr"),tt=x("td"),et=x("div"),pt=0;pt{u()}),t.$set=(t=>{"active"in t&&n("active",c=t.active)}),t.$$.update=((t={total:1,count:1})=>{(t.total||t.count)&&n("pages",o=Math.ceil(s/je))}),{active:c,key:l,keys:a,json:i,total:s,page:d,getall:u,showkey:v,next:function(){d1&&n("page",d-=1),u()},pages:o,input_input_handler:function(){l=this.value,n("key",l)},keypress_handler:function(t){13==(t.keyCode||t.which)&&v(l)},click_handler:function(){return v(l)},click_handler_1:function({v:t}){return v(t)}}}class De extends ht{constructor(t){super(),pt(this,t,Oe,Se,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Le(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ne(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ve(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ie(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Te(t,e,n){const o=Object.create(t);return o.sc=e[n],o}function Me(t,e,n){const o=Object.create(t);return o.vn=e[n],o}function Be(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt,ut,vt,ft,pt,ht,mt,gt,bt,wt,$t,xt,yt,kt,_t,Ct,St,jt,Ot,Dt,Lt,Nt,Vt,It,Tt,Mt,Bt,Et,qt,Rt,Ft,At,Gt,Ht,zt,Ut,Jt,Pt,Kt,Xt,Qt,Wt,Yt,Zt,te,ee,ne,oe,ce,le,ae=[],ie=new Map,re=[],se=new Map,de=[],ue=new Map,ve=[],fe=new Map,pe=[],he=new Map,me=[],ge=new Map,be=t.viewnames;const we=t=>t.vn;for(var $e=0;$et.sc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$eFull Text Columns : Case Insensitive Columns : String Index Length : No Indexing Columns :",Jt=k(),Pt=x("tr"),Kt=x("td"),Xt=x("div"),$e=0;$et.select_change_handler.call(c)),O(c,"width","200px"),O(c,"font-size","larger"),S(p,"class","bold svelte-mdntjp"),S(I,"class","bold svelte-mdntjp"),S(F,"class","bold svelte-mdntjp"),S(P,"class","bold svelte-mdntjp"),S(et,"class","bold svelte-mdntjp"),S(rt,"class","bold svelte-mdntjp"),S(gt,"class","bold svelte-mdntjp"),S(_t,"class","bold svelte-mdntjp"),S(Nt,"class","bold svelte-mdntjp"),S(Rt,"class","svelte-mdntjp"),S(At,"class","svelte-mdntjp"),S(qt,"class","schema_def svelte-mdntjp"),S(Xt,"class","ListBox svelte-mdntjp"),S(Kt,"width","25%"),S(Yt,"class","ListBox svelte-mdntjp"),S(Wt,"width","25%"),S(ee,"class","ListBox svelte-mdntjp"),S(te,"width","25%"),S(ce,"class","ListBox svelte-mdntjp"),S(oe,"width","25%"),S(zt,"width","100%"),S(u,"class","View"),S(e,"class","tab-content"),N(e,"active",t.active),le=[C(c,"change",t.select_change_handler),C(c,"change",t.schemachanged)]},m(l,w){for(b(l,e,w),g(e,n),g(e,o),g(e,c),$e=0;$e{window.GET("RaptorDB/GetViews",function(t){var e=[];t.Rows.forEach(t=>e.push(t.Name)),n("viewnames",l=e),e.length>0&&(n("viewname",c=e[0]),$())})}),t.$set=(t=>{"active"in t&&n("active",o=t.active)}),{active:o,viewname:c,viewnames:l,schema:a,FullTextColumns:i,CaseInsensitiveColumns:r,StringIndexLength:s,NoIndexingColumns:d,Name:u,Description:v,Version:f,isPrimaryList:p,isActive:h,DeleteBeforeInsert:m,BackgroundIndexing:g,ConsistentSaveToThisView:b,TransactionMode:w,schemachanged:$,select_change_handler:function(){c=L(this),n("viewname",c),n("viewnames",l)}}}class Ue extends ht{constructor(t){super(),pt(this,t,ze,He,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Je(t){var e,n,o,c,l,a;return{c(){e=x("div"),(n=x("h2")).textContent="raptordb.config",o=k(),c=x("div"),l=x("pre"),a=y(t.configs),S(l,"class","JSON"),S(c,"class","JSONArea"),S(e,"class","tab-content"),N(e,"active",t.active)},m(t,i){b(t,e,i),g(e,n),g(e,o),g(e,c),g(c,l),g(l,a)},p(t,n){t.configs&&j(a,n.configs),t.active&&N(e,"active",n.active)},d(t){t&&w(e)}}}function Pe(e){var n,o=e.active&&Je(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=Je(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function Ke(t,e,n){let{active:o=!1}=e,c="";return R(()=>{window.LOAD("/raptordb/action?getconfigs",function(t){n("configs",c=t)})}),t.$set=(t=>{"active"in t&&n("active",o=t.active)}),{active:o,configs:c}}class Xe extends ht{constructor(t){super(),pt(this,t,Ke,Pe,i,["active"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Qe(t){const e=t-1;return e*e*e+1}function We(t,{delay:e=0,duration:n=400}){const o=+getComputedStyle(t).opacity;return{delay:e,duration:n,css:t=>`opacity: ${t*o}`}}function Ye(t,{delay:e=0,duration:n=400,easing:o=Qe,x:c=0,y:l=0,opacity:a=0}){const i=getComputedStyle(t),r=+i.opacity,s="none"===i.transform?"":i.transform,d=r*(1-a);return{delay:e,duration:n,easing:o,css:(t,e)=>`\n\t\t\ttransform: ${s} translate(${(1-t)*c}px, ${(1-t)*l}px);\n\t\t\topacity: ${r-d*e}`}}function Ze(t){var e,n,o,c;const l=t.$$slots.default,a=r(l,t,null);return{c(){e=x("button"),a&&a.c(),S(e,"class",n=t.mode+" "+t.color+" svelte-g32zaw"),S(e,"type",t.type),e.disabled=t.disabled,c=C(e,"click",t.click_handler)},l(t){a&&a.l(button_nodes)},m(t,n){b(t,e,n),a&&a.m(e,null),o=!0},p(t,c){a&&a.p&&t.$$scope&&a.p(d(l,c,t,null),s(l,c,null)),o&&!t.mode&&!t.color||n===(n=c.mode+" "+c.color+" svelte-g32zaw")||S(e,"class",n),o&&!t.type||S(e,"type",c.type),o&&!t.disabled||(e.disabled=c.disabled)},i(t){o||(at(a,t),o=!0)},o(t){it(a,t),o=!1},d(t){t&&w(e),a&&a.d(t),c()}}}function tn(t){var e,n;const o=t.$$slots.default,c=r(o,t,null);return{c(){e=x("a"),c&&c.c(),S(e,"href",t.href),S(e,"class","svelte-g32zaw")},l(t){c&&c.l(a_nodes)},m(t,o){b(t,e,o),c&&c.m(e,null),n=!0},p(t,l){c&&c.p&&t.$$scope&&c.p(d(o,l,t,null),s(o,l,null)),n&&!t.href||S(e,"href",l.href)},i(t){n||(at(c,t),n=!0)},o(t){it(c,t),n=!1},d(t){t&&w(e),c&&c.d(t)}}}function en(t){var e,n,o,c,l=[tn,Ze],a=[];function i(t){return t.href?0:1}return e=i(t),n=a[e]=l[e](t),{c(){n.c(),o=_()},m(t,n){a[e].m(t,n),b(t,o,n),c=!0},p(t,c){var r=e;(e=i(c))===r?a[e].p(t,c):(ct(),it(a[r],1,1,()=>{a[r]=null}),lt(),(n=a[e])||(n=a[e]=l[e](c)).c(),at(n,1),n.m(o.parentNode,o))},i(t){c||(at(n),c=!0)},o(t){it(n),c=!1},d(t){a[e].d(t),t&&w(o)}}}function nn(t,e,n){let{type:o="button",href:c=null,mode:l=null,color:a=null,disabled:i=!1}=e,{$$slots:r={},$$scope:s}=e;return t.$set=(t=>{"type"in t&&n("type",o=t.type),"href"in t&&n("href",c=t.href),"mode"in t&&n("mode",l=t.mode),"color"in t&&n("color",a=t.color),"disabled"in t&&n("disabled",i=t.disabled),"$$scope"in t&&n("$$scope",s=t.$$scope)}),{type:o,href:c,mode:l,color:a,disabled:i,click_handler:function(e){A(t,e)},$$slots:r,$$scope:s}}class on extends ht{constructor(t){super(),pt(this,t,nn,en,i,["type","href","mode","color","disabled"])}}const cn=()=>({}),ln=()=>({});function an(t){var e;return{c(){e=y("Close")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function rn(t){var e,n,o,c,l,a,i,u,v,f,p,h,m;const $=t.$$slots.default,_=r($,t,null),O=t.$$slots.footer,D=r(O,t,ln);var L=new on({props:{$$slots:{default:[an]},$$scope:{ctx:t}}});return L.$on("click",t.closeModal),{c(){e=x("div"),o=k(),c=x("div"),l=x("h1"),a=y(t.title),i=k(),u=x("div"),_&&_.c(),v=k(),f=x("footer"),D||L.$$.fragment.c(),D&&D.c(),S(e,"class","modal-backdrop svelte-f1bfsy"),S(l,"class","svelte-f1bfsy"),S(u,"class","content svelte-f1bfsy"),S(f,"class","svelte-f1bfsy"),S(c,"class","modal svelte-f1bfsy"),m=C(e,"click",t.closeModal)},l(t){_&&_.l(div1_nodes),D&&D.l(footer_nodes)},m(t,n){b(t,e,n),b(t,o,n),b(t,c,n),g(c,l),g(l,a),g(c,i),g(c,u),_&&_.m(u,null),g(c,v),g(c,f),D?D.m(f,null):vt(L,f,null),h=!0},p(t,e){if(h&&!t.title||j(a,e.title),_&&_.p&&t.$$scope&&_.p(d($,e,t,null),s($,e,null)),!D){var n={};t.$$scope&&(n.$$scope={changed:t,ctx:e}),L.$set(n)}D&&D.p&&t.$$scope&&D.p(d(O,e,t,cn),s(O,e,ln))},i(t){h||(W(()=>{n||(n=rt(e,We,{},!0)),n.run(1)}),at(_,t),at(L.$$.fragment,t),at(D,t),W(()=>{p||(p=rt(c,Ye,{y:300,duration:200},!0)),p.run(1)}),h=!0)},o(t){n||(n=rt(e,We,{},!1)),n.run(0),it(_,t),it(L.$$.fragment,t),it(D,t),p||(p=rt(c,Ye,{y:300,duration:200},!1)),p.run(0),h=!1},d(t){t&&(w(e),n&&n.end(),w(o),w(c)),_&&_.d(t),D||ft(L),D&&D.d(t),t&&p&&p.end(),m()}}}function sn(t,e,n){let{title:o}=e;const c=F();let{$$slots:l={},$$scope:a}=e;return t.$set=(t=>{"title"in t&&n("title",o=t.title),"$$scope"in t&&n("$$scope",a=t.$$scope)}),{title:o,closeModal:function(){c("cancel")},$$slots:l,$$scope:a}}class dn extends ht{constructor(t){super(),pt(this,t,sn,rn,i,["title"])}}function un(t,e,n){const o=Object.create(t);return o.tab=e[n],o}function vn(t){var e,n;function o(){return t.click_handler(t)}return{c(){(e=x("label")).textContent="X",S(e,"id","close"),S(e,"class","svelte-obficw"),n=C(e,"click",o)},m(t,n){b(t,e,n)},p(e,n){t=n},d(t){t&&w(e),n()}}}function fn(t,e){var n,o,c,l,a,i,r,s,d=e.tab.title,u="Help"!==e.tab.title&&vn(e);function v(){return e.click_handler_1(e)}return{key:t,first:null,c(){n=x("li"),o=x("div"),c=x("label"),l=y(d),a=k(),u&&u.c(),i=k(),S(o,"class","svelte-obficw"),S(n,"id",r=e.tab.id),S(n,"class","svelte-obficw"),N(n,"active",e.tab.id===e.activetabid),s=C(n,"click",v),this.first=n},m(t,e){b(t,n,e),g(n,o),g(o,c),g(c,l),g(o,a),u&&u.m(o,null),g(n,i)},p(t,c){e=c,t.tabs&&d!==(d=e.tab.title)&&j(l,d),"Help"!==e.tab.title?u||((u=vn(e)).c(),u.m(o,null)):u&&(u.d(1),u=null),t.tabs&&r!==(r=e.tab.id)&&S(n,"id",r),(t.tabs||t.activetabid)&&N(n,"active",e.tab.id===e.activetabid)},d(t){t&&w(n),u&&u.d(),s()}}}function pn(t){var e,n=new dn({props:{title:t.title,$$slots:{default:[wn],footer:[bn]},$$scope:{ctx:t}}});return n.$on("cancel",t.cancel_handler),{c(){n.$$.fragment.c()},m(t,o){vt(n,t,o),e=!0},p(t,e){var o={};t.title&&(o.title=e.title),(t.$$scope||t.modalyes)&&(o.$$scope={changed:t,ctx:e}),n.$set(o)},i(t){e||(at(n.$$.fragment,t),e=!0)},o(t){it(n.$$.fragment,t),e=!1},d(t){ft(n,t)}}}function hn(t){var e,n=new on({props:{type:"button",$$slots:{default:[mn]},$$scope:{ctx:t}}});return n.$on("click",t.click_handler_2),{c(){n.$$.fragment.c()},m(t,o){vt(n,t,o),e=!0},i(t){e||(at(n.$$.fragment,t),e=!0)},o(t){it(n.$$.fragment,t),e=!1},d(t){ft(n,t)}}}function mn(t){var e;return{c(){e=y("Yes")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function gn(t){var e;return{c(){e=y("Cancel")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function bn(t){var e,n,o,c=t.modalyes&&hn(t),l=new on({props:{type:"button",mode:"outline",$$slots:{default:[gn]},$$scope:{ctx:t}}});return l.$on("click",t.click_handler_3),{c(){e=x("div"),c&&c.c(),n=k(),l.$$.fragment.c(),S(e,"slot","footer")},m(t,a){b(t,e,a),c&&c.m(e,null),g(e,n),vt(l,e,null),o=!0},p(t,o){o.modalyes?c?at(c,1):((c=hn(o)).c(),at(c,1),c.m(e,n)):c&&(ct(),it(c,1,1,()=>{c=null}),lt());var a={};t.$$scope&&(a.$$scope={changed:t,ctx:o}),l.$set(a)},i(t){o||(at(c),at(l.$$.fragment,t),o=!0)},o(t){it(c),it(l.$$.fragment,t),o=!1},d(t){t&&w(e),c&&c.d(),ft(l)}}}function wn(e){return{c:t,m:t,p:t,i:t,o:t,d:t}}function $n(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p=[],h=new Map,m=new yt({});m.$on("navclick",t.navclick_handler);var $=t.tabs;const y=t=>t.tab.id;for(var j=0;j<$.length;j+=1){let e=un(t,$,j),n=y(e);h.set(n,p[j]=fn(n,e))}var O=t.showmsg&&pn(t);return{c(){for(e=x("div"),n=x("div"),m.$$.fragment.c(),o=k(),c=x("div"),l=x("ul"),(a=x("span")).textContent="X",i=k(),j=0;j{O=null}),lt())},i(t){v||(at(m.$$.fragment,t),at(O),v=!0)},o(t){it(m.$$.fragment,t),it(O),v=!1},d(t){for(t&&w(e),ft(m),j=0;j{var o=function(t,e,n){var o=null;switch(t){case"help":o=new St({target:document.getElementById(e)});break;case"query":(o=new Qt({target:document.getElementById(e)})).$on("showdoc",t=>{r({name:"docview",description:"View"},t.detail)});break;case"sysinfo":o=new ne({target:document.getElementById(e)});break;case"docsearch":o=new de({target:document.getElementById(e)});break;case"docview":(o=new he({target:document.getElementById(e),props:{docid:n||null}})).$on("showrevs",t=>{r({name:"dochistory",description:"History"},t.detail)});break;case"dochistory":o=new ye({target:document.getElementById(e),props:{docid:n||null}});break;case"hfbrowser":o=new De({target:document.getElementById(e)});break;case"schema":o=new Ue({target:document.getElementById(e)});break;case"sysconfig":o=new Xe({target:document.getElementById(e)})}return o}(t.name,i,e);n("tabs",c=[...c,{title:t.description,id:i,active:!0,obj:o}]),d(i)})}function s(t){r(t)}function d(t){n("activetabid",o=t),c.forEach(e=>{e.active=e.id==t,e.obj.active=e.active}),n("tabs",c)}function u(t){var e=c.findIndex(e=>e.id===t);d(c[--e].id),function(t){var e=c.find(e=>e.id===t);null!==e&&null!==e.obj&&(e.obj.$destroy(),n("tabs",c=c.filter(e=>e.id!==t)),document.querySelector("#mainid #"+t).remove())}(t)}function v(){n("showmsg",l=!1),c.forEach(t=>{"Help"!==t.title&&u(t.id)})}return R(()=>{r({name:"help",description:"Help"}),Q().then(()=>d(c[0].id))}),{activetabid:o,tabs:c,showmsg:l,title:a,modalyes:i,showtab:s,changetab:d,closetab:u,closealltabs:v,closeall:function(){n("showmsg",l=!0),n("modalyes",i=!1),1==c.length?n("title",a="No Tabs to close"):(n("title",a="Do you want to close all the tabs?"),n("modalyes",i=!0))},navclick_handler:function(t){return s(t.detail)},click_handler:function({tab:t}){return u(t.id)},click_handler_1:function({tab:t}){return d(t.id)},click_handler_2:function(){return v()},click_handler_3:function(){const t=l=!1;return n("showmsg",l),t},cancel_handler:function(){const t=l=!1;return n("showmsg",l),t}}}const yn=new class extends ht{constructor(t){super(),pt(this,t,xn,$n,i,[])}}({target:document.body});return window.ServerURL=document.location.protocol+"//"+document.location.hostname+":"+document.location.port+"/",window.GET=function(t,e,n){var o=new XMLHttpRequest;o.open("GET",window.ServerURL+t),o.onreadystatechange=function(){if(1!==o.readyState&&4===o.readyState)if(o.status<400){var t=null;t=""!==o.responseText?JSON.parse(o.responseText):"",e&&e(t)}else 401===o.status?(o.abort(),e&&e("")):null!==n&&n(o.responseText)},o.onerror=function(t){console.log(t)},o.withCredentials=!1,o.send()},window.LOAD=function(t,e,n){var o=new XMLHttpRequest;o.open("GET",window.ServerURL+t),o.onreadystatechange=function(){if(1!==o.readyState&&4===o.readyState)if(o.status<400){var t=null;t=""!==o.responseText?o.responseText:"",e&&e(t)}else 401===o.status?(o.abort(),e&&e("")):null!==n&&n(o.responseText)},o.onerror=function(t){console.log(t)},o.withCredentials=!1,o.send()},yn}(); ================================================ FILE: RaptorDB/WEB/global.css ================================================ body { font-family: sans-serif; color: #222; background: #414141; } .tab-content { background-color: white; padding: 5px; padding-top: 15px; min-height: 90vh; } .LogArea { border-style: solid; border-color: gray; background-color: #e0e0e0; padding: 5px; } .LogItems { font-family: "Courier New", monospace; white-space: pre-wrap; font-size: smaller; } .JSONArea { border-style: solid; border-color: gray; background-color: white; padding: 5px; min-height: 500px; } .JSON { font-family: "Courier New", monospace; white-space: pre-wrap; font-size: smaller; } ================================================ FILE: RaptorDB/WEB/index.html ================================================ RaptorDB WebStudio ================================================ FILE: RaptorDB/cron/CronDaemon.cs ================================================ using System; using System.Collections.Generic; using System.Timers; using System.Threading; namespace RaptorDB { public class CronDaemon { private readonly System.Timers.Timer timer = new System.Timers.Timer(30000); private readonly List cron_jobs = new List(); private DateTime _last= DateTime.Now; public CronDaemon() { timer.AutoReset = true; timer.Elapsed += timer_elapsed; timer.Start(); } public void AddJob(string schedule, ThreadStart action) { var cj = new CronJob(schedule, action); cron_jobs.Add(cj); } //public void RemoveJob(string schedule)//, ThreadStart action) //{ // var f = cron_jobs.Find((x) => { return x._cron_schedule._expression == schedule; }); // if(f!=null) // cron_jobs.Remove(f); //} //public void Start() //{ // timer.Start(); //} public void Stop() { timer.Stop(); foreach (CronJob job in cron_jobs) job.abort(); } private void timer_elapsed(object sender, ElapsedEventArgs e) { if (DateTime.Now.Minute != _last.Minute) { _last = DateTime.Now; foreach (CronJob job in cron_jobs) job.execute(DateTime.Now); } } } } ================================================ FILE: RaptorDB/cron/CronJob.cs ================================================ using System; using System.Threading; namespace RaptorDB { internal class CronJob { internal readonly CronSchedule _cron_schedule = new CronSchedule(); private readonly ThreadStart _thread_start; internal Thread _thread; public CronJob(string schedule, ThreadStart thread_start) { _cron_schedule = new CronSchedule(schedule); _thread_start = thread_start; _thread = new Thread(thread_start); } private object _lock = new object(); public void execute(DateTime date_time) { lock (_lock) { if (!_cron_schedule.isTime(date_time)) return; if (_thread.ThreadState == ThreadState.Running) return; _thread = new Thread(_thread_start); _thread.Start(); } } public void abort() { try { _thread.Abort(); } catch { } } } } ================================================ FILE: RaptorDB/cron/CronSchedule.cs ================================================ using System; using System.Collections.Generic; using System.Text.RegularExpressions; namespace RaptorDB { internal class CronSchedule { #region Readonly Class Members readonly static Regex divided_regex = new Regex(@"(\*/\d+)"); readonly static Regex range_regex = new Regex(@"(\d+\-\d+)\/?(\d+)?"); readonly static Regex wild_regex = new Regex(@"(\*)"); readonly static Regex list_regex = new Regex(@"(((\d+,)*\d+)+)"); readonly static Regex validation_regex = new Regex(divided_regex + "|" + range_regex + "|" + wild_regex + "|" + list_regex); #endregion #region Private Instance Members internal readonly string _expression; public List minutes; public List hours; public List days_of_month; public List months; public List days_of_week; #endregion #region Public Constructors public CronSchedule() { } public CronSchedule(string expressions) { this._expression = expressions; generate(); } #endregion #region Public Methods private bool isValid() { return isValid(this._expression); } public bool isValid(string expression) { MatchCollection matches = validation_regex.Matches(expression); return matches.Count > 0;//== 5; } public bool isTime(DateTime date_time) { return minutes.Contains(date_time.Minute) && hours.Contains(date_time.Hour) && days_of_month.Contains(date_time.Day) && months.Contains(date_time.Month) && days_of_week.Contains((int)date_time.DayOfWeek); } private void generate() { if (!isValid()) return; MatchCollection matches = validation_regex.Matches(this._expression); generate_minutes(matches[0].ToString()); if (matches.Count > 1) generate_hours(matches[1].ToString()); else generate_hours("*"); if (matches.Count > 2) generate_days_of_month(matches[2].ToString()); else generate_days_of_month("*"); if (matches.Count > 3) generate_months(matches[3].ToString()); else generate_months("*"); if (matches.Count > 4) generate_days_of_weeks(matches[4].ToString()); else generate_days_of_weeks("*"); } private void generate_minutes(string match) { this.minutes = generate_values(match, 0, 60); } private void generate_hours(string match) { this.hours = generate_values(match, 0, 24); } private void generate_days_of_month(string match) { this.days_of_month = generate_values(match, 1, 32); } private void generate_months(string match) { this.months = generate_values(match, 1, 13); } private void generate_days_of_weeks(string match) { this.days_of_week = generate_values(match, 0, 7); } private List generate_values(string configuration, int start, int max) { if (divided_regex.IsMatch(configuration)) return divided_array(configuration, start, max); if (range_regex.IsMatch(configuration)) return range_array(configuration); if (wild_regex.IsMatch(configuration)) return wild_array(configuration, start, max); if (list_regex.IsMatch(configuration)) return list_array(configuration); return new List(); } private List divided_array(string configuration, int start, int max) { if (!divided_regex.IsMatch(configuration)) return new List(); List ret = new List(); string[] split = configuration.Split("/".ToCharArray()); int divisor = int.Parse(split[1]); for (int i = start; i < max; ++i) if (i % divisor == 0) ret.Add(i); return ret; } private List range_array(string configuration) { if (!range_regex.IsMatch(configuration)) return new List(); List ret = new List(); string[] split = configuration.Split("-".ToCharArray()); int start = int.Parse(split[0]); int end = 0; if (split[1].Contains("/")) { split = split[1].Split("/".ToCharArray()); end = int.Parse(split[0]); int divisor = int.Parse(split[1]); for (int i = start; i < end; ++i) if (i % divisor == 0) ret.Add(i); return ret; } else end = int.Parse(split[1]); for (int i = start; i <= end; ++i) ret.Add(i); return ret; } private List wild_array(string configuration, int start, int max) { if (!wild_regex.IsMatch(configuration)) return new List(); List ret = new List(); for (int i = start; i < max; ++i) ret.Add(i); return ret; } private List list_array(string configuration) { if (!list_regex.IsMatch(configuration)) return new List(); List ret = new List(); string[] split = configuration.Split(",".ToCharArray()); foreach (string s in split) ret.Add(int.Parse(s)); return ret; } #endregion } } ================================================ FILE: RaptorDB.Common/DataTypes.cs ================================================ using System; using System.Collections.Generic; namespace RaptorDB { /// /// Result of queries /// OK : T = Query with data, F = EX has the exception /// Rows : query rows /// public class Result { public Result() { } public Result(bool ok) { OK = ok; } public Result(bool ok, Exception ex) { OK = ok; EX = ex; } /// /// T=Values return, F=exceptions occurred /// public bool OK { get; set; } public Exception EX { get; set; } /// /// Total number of rows of the query /// public int TotalCount { get; set; } /// /// Rows returned /// public int Count { get; set; } public List Rows { get; set; } public string Title { get; set; } // FEATURE : data pending in results ///// ///// Data is being indexed, so results will not reflect all documents ///// //public bool DataPending { get; set; } } /// /// Base for row schemas : implements a docid property and is bindable /// public abstract class RDBSchema : BindableFields { public Guid docid; } } // no namespace -> available to all public static class RDBExtension { /// /// RDB Between checking for dates /// /// /// Must be yyyy-mm-dd /// Must be yyyy-mm-dd /// public static bool Between(this DateTime value, string fromdate, string todate) { return true; } public static bool Between(this T value, T from, T to) { return true; } public static bool In(this byte value, params byte[] values) { return true; } public static bool In(this T value, params T[] values) { return true; } } ================================================ FILE: RaptorDB.Common/FieldDescriptor.cs ================================================ using System; using System.Reflection; using System.ComponentModel; namespace RaptorDB { internal class FieldPropertyDescriptor : PropertyDescriptor { private FieldInfo _field; public FieldPropertyDescriptor(FieldInfo field) : base(field.Name, (Attribute[])field.GetCustomAttributes(typeof(Attribute), true)) { _field = field; } public FieldInfo Field { get { return _field; } } public override bool Equals(object obj) { FieldPropertyDescriptor other = obj as FieldPropertyDescriptor; return other != null && other._field.Equals(_field); } public override int GetHashCode() { return _field.GetHashCode(); } public override bool IsReadOnly { get { return false; } } public override void ResetValue(object component) { } public override bool CanResetValue(object component) { return false; } public override bool ShouldSerializeValue(object component) { return true; } public override Type ComponentType { get { return _field.DeclaringType; } } public override Type PropertyType { get { return _field.FieldType; } } public override object GetValue(object component) { return _field.GetValue(component); } public override void SetValue(object component, object value) { _field.SetValue(component, value); OnValueChanged(component, EventArgs.Empty); } } public abstract class BindableFields : ICustomTypeDescriptor { object ICustomTypeDescriptor.GetPropertyOwner(PropertyDescriptor pd) { return this; } AttributeCollection ICustomTypeDescriptor.GetAttributes() { return TypeDescriptor.GetAttributes(this, true); } string ICustomTypeDescriptor.GetClassName() { return TypeDescriptor.GetClassName(this, true); } string ICustomTypeDescriptor.GetComponentName() { return TypeDescriptor.GetComponentName(this, true); } TypeConverter ICustomTypeDescriptor.GetConverter() { return TypeDescriptor.GetConverter(this, true); } EventDescriptor ICustomTypeDescriptor.GetDefaultEvent() { return TypeDescriptor.GetDefaultEvent(this, true); } PropertyDescriptor ICustomTypeDescriptor.GetDefaultProperty() { return TypeDescriptor.GetDefaultProperty(this, true); } object ICustomTypeDescriptor.GetEditor(Type editorBaseType) { return TypeDescriptor.GetEditor(this, editorBaseType, true); } EventDescriptorCollection ICustomTypeDescriptor.GetEvents(Attribute[] attributes) { return TypeDescriptor.GetEvents(this, attributes, true); } EventDescriptorCollection ICustomTypeDescriptor.GetEvents() { return TypeDescriptor.GetEvents(this, true); } private PropertyDescriptorCollection _propCache; private FilterCache _filterCache; PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties() { return ((ICustomTypeDescriptor)this).GetProperties(null); } PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties(Attribute[] attributes) { bool filtering = (attributes != null && attributes.Length > 0); PropertyDescriptorCollection props = _propCache; FilterCache cache = _filterCache; // Use a cached version if possible if (filtering && cache != null && cache.IsValid(attributes)) { return cache.FilteredProperties; } else if (!filtering && props != null) { return props; } // Create the property collection and filter props = new PropertyDescriptorCollection(null); foreach (PropertyDescriptor prop in TypeDescriptor.GetProperties(this, attributes, true)) { props.Add(prop); } foreach (FieldInfo field in GetType().GetFields()) { FieldPropertyDescriptor fieldDesc = new FieldPropertyDescriptor(field); if (!filtering || fieldDesc.Attributes.Contains(attributes)) props.Add(fieldDesc); } // Store the computed properties if (filtering) { cache = new FilterCache(); cache.Attributes = attributes; cache.FilteredProperties = props; _filterCache = cache; } else _propCache = props; return props; } private class FilterCache { public Attribute[] Attributes; public PropertyDescriptorCollection FilteredProperties; public bool IsValid(Attribute[] other) { if (other == null || Attributes == null) return false; if (Attributes.Length != other.Length) return false; for (int i = 0; i < other.Length; i++) { if (!Attributes[i].Match(other[i])) return false; } return true; } } } } ================================================ FILE: RaptorDB.Common/IRaptorDB.cs ================================================ using System; using System.Collections.Generic; using System.Linq.Expressions; namespace RaptorDB.Common { public delegate List ServerSideFunc(IRaptorDB rap, string filter); public delegate List ServerSideFuncWithArgs(IRaptorDB rap, string filter, params object[] args); public class HistoryInfo { public int Version; public DateTime ChangeDate; } /// /// High frequency mode Key/Value store with recycled storage file. /// Use for rapid saves of the same key. /// Views are not effected by saves in this storage. /// NOTE : You do not have history of changes in this storage. /// public interface IKeyStoreHF { object GetObjectHF(string key); bool SetObjectHF(string key, object obj); bool DeleteKeyHF(string key); int CountHF(); bool ContainsHF(string key); string[] GetKeysHF(); void CompactStorageHF(); int Increment(string key, int amount); decimal Increment(string key, decimal amount); int Decrement(string key, int amount); decimal Decrement(string key, decimal amount); //T Increment(string key, T amount); //T Decrement(string key, T amount); //IEnumerable EnumerateObjects(); //string[] SearchKeys(string contains); // FIX : implement } public interface IRaptorDB { /// /// Save Bytes (files) to RptorDB storage /// /// /// /// bool SaveBytes(Guid fileID, byte[] bytes); /// /// Save a Document to RaptorDB /// /// /// /// /// bool Save(Guid docID, T document); /// /// Query all data in a view /// /// /// Result Query(string viewname); /// /// Query all data in a view with paging /// /// /// /// /// Result Query(string viewname, int start, int count); /// /// Query a View with a string filter /// /// /// /// Result Query(string viewname, string filter); /// /// Query a View with a string filter with paging /// /// /// /// /// /// Result Query(string viewname, string filter, int start, int count); /// /// Query a view with filter, paging and sorting /// /// /// /// /// /// Result Query(string viewname, string filter, int start, int count, string orderby); /// /// Count all data associated with View name /// /// /// int Count(string viewname); /// /// Count all data associated with View name and string filter /// /// /// /// int Count(string viewname, string filter); /// /// Fetch a Document /// /// /// object Fetch(Guid docID); /// /// Fetch a Document when you know the type /// /// /// T Fetch(Guid docID) where T : class; /// /// Fetch a file bytes /// /// /// byte[] FetchBytes(Guid fileID); ///// ///// Shutdown RaptorDB and flush all data to disk ///// //void Shutdown(); /// /// Backup the document storage file incrementally to "Backup" folder /// /// True = done bool Backup(); /// /// Start background restore of backups in the "Restore" folder /// void Restore(); /// /// Delete a Document /// /// /// bool Delete(Guid docid); /// /// Delete a File /// /// /// bool DeleteBytes(Guid fileid); /// /// Add users /// /// /// /// /// bool AddUser(string username, string oldpassword, string newpassword); /// /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing /// /// /// object[] ServerSide(ServerSideFunc func, string filter); /// /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing /// /// /// object[] ServerSide(ServerSideFunc func, Expression> filter); /// /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing /// /// /// object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args); /// /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing /// /// /// object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args); /// /// Full text search the entire original document /// /// /// int[] FullTextSearch(string filter); // new query model /// /// Query a view with linq filter /// /// Use the Row Schema type for your view /// /// Result Query(Expression> filter); /// /// Query a view with paging /// /// Use the Row Schema type for your view /// /// /// /// Result Query(Expression> filter, int start, int count); /// /// Query a view with linq filter, paging and sorting /// /// Use the Row Schema type for your view /// /// /// /// /// Result Query(Expression> filter, int start, int count, string orderby); /// /// Query a view with a string filter /// /// Use the Row Schema type for your view /// /// Result Query(string filter); /// /// Query a view with string filter and paging /// /// Use the Row Schema type for your view /// /// /// /// Result Query(string filter, int start, int count); /// /// Query a view with string filter, paging and sorting /// /// Use the Row Schema type for your view /// /// /// /// /// Result Query(string filter, int start, int count, string orderby); /// /// Count rows with a linq filter /// /// /// /// int Count(Expression> filter); /// /// Fetch the change history for a document /// /// /// int[] FetchHistory(Guid docid); /// /// Fetch the change history for a document with dates /// /// /// HistoryInfo[] FetchHistoryInfo(Guid docid); /// /// Fetch a change history for a file /// /// /// int[] FetchBytesHistory(Guid fileid); /// /// Fetch a change history for a file with dates /// /// /// HistoryInfo[] FetchBytesHistoryInfo(Guid docid); /// /// Fetch the specific document version /// /// /// object FetchVersion(int versionNumber); /// /// Fetch the specific file version /// /// /// byte[] FetchBytesVersion(int versionNumber); /// /// Delete rows from a view /// /// /// /// Number of rows deleted int ViewDelete(Expression> filter); /// /// Delete rows from a view /// /// /// /// Number of rows deleted int ViewDelete(string viewname, string filter); /// /// Insert directly into a view /// /// /// /// /// bool ViewInsert(Guid id, TRowSchema row); /// /// Insert directly into a view /// /// /// /// /// bool ViewInsert(string viewname, Guid id, object row); /// /// Get the number of documents in the storage file regardless of versions /// /// long DocumentCount(); /// /// High frequency mode Key/Value store with recycled storage file. /// Use for rapid saves of the same key. /// Views are not effected by saves in this storage. /// NOTE : You do not have history of changes in this storage. /// IKeyStoreHF GetKVHF(); void FreeMemory(); void Shutdown(); void SaveToDocsOnViewInsert(bool yes); } } ================================================ FILE: RaptorDB.Common/Interfaces.cs ================================================ using System; using System.Collections.Generic; using System.Linq.Expressions; using RaptorDB.Common; namespace RaptorDB { //public static class RDBExtensions //{ // ///// // ///// For RaptorDB optimized range queries // ///// // ///// // ///// // ///// // ///// // ///// // //public static bool Between(this T obj, T from, T to) // //{ // // return true; // //} // ///// // ///// For RaptorDB full text search queries // ///// // ///// // ///// // ///// // //public static bool Contains(this string obj, string what) // //{ // // return true; // //} //} /// /// Used for normal string columns /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] public class CaseInsensitiveAttribute : Attribute { } /// /// Used for the indexer -> hOOt full text indexing /// [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] public class FullTextAttribute : Attribute { } /// /// Used for declaring view extensions DLL's /// [AttributeUsage(AttributeTargets.Class)] public class RegisterViewAttribute : Attribute { } /// /// Index file max string length size in UTF8 (Default = 60) /// [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)] public class StringIndexLength : Attribute { public StringIndexLength() { Length = 60; // default } public StringIndexLength(byte length) { Length = length; } public byte Length; } public interface IQueryInterface { /// /// Log messages /// /// void Log(string message); /// /// Count all data associated with the Documnet Type or the View Type with a string filter /// /// /// int Count(string viewname); /// /// Count all data associated with View name and string filter /// /// /// /// int Count(string ViewName, string Filter); /// /// Fetch a document by it's Guid /// /// /// object Fetch(Guid guid); /// /// Fetch a document by it's Guid /// /// /// T Fetch(Guid guid) where T : class; // new query model Result Query(Expression> Filter); Result Query(Expression> Filter, int start, int count); Result Query(string Filter); Result Query(string Filter, int start, int count); int Count(Expression> Filter); } public interface IMapAPI : IQueryInterface { /// /// Emit values, the ordering must match the view schema /// /// /// void Emit(Guid docid, params object[] data); /// /// Emits the object matching the view schema, you must make sure the object property names match the row schema /// /// /// /// void EmitObject(Guid docid, T doc); /// /// Roll back the transaction if the primary view is in transaction mode /// void RollBack(); /// /// Get the next row number for this view /// /// int NextRowNumber(); IKeyStoreHF GetKVHF(); //void EmitRow(Guid docid, V row); } public interface IClientHandler { bool GenerateClientData(IQueryInterface api, string username, List DocsToSend); } public enum COMMANDS { Save, SaveBytes, QueryType, QueryStr, Fetch, FetchBytes, Backup, Delete, DeleteBytes, Restore, AddUser, ServerSide, FullText, CountType, CountStr, GCount, DocHistory, FileHistory, FetchVersion, FetchFileVersion, CheckAssembly, FetchHistoryInfo, FetchByteHistoryInfo, ViewDelete, ViewDelete_t, ViewInsert, ViewInsert_t, DocCount, GetObjectHF, SetObjectHF, DeleteKeyHF, CountHF, ContainsHF, GetKeysHF, CompactStorageHF, IncrementHF, DecrementHF, ServerSideWithArgs, FreeMemory } } ================================================ FILE: RaptorDB.Common/LINQString.cs ================================================ using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Text; namespace RaptorDB { public class LINQString : ExpressionVisitor { public StringBuilder sb = new StringBuilder(); protected override Expression VisitBinary(BinaryExpression b) { sb.Append("("); this.Visit(b.Left); ExpressionType t = b.NodeType; switch (b.NodeType) { case ExpressionType.AndAlso: case ExpressionType.And: sb.Append(" AND "); break; case ExpressionType.OrElse: case ExpressionType.Or: sb.Append(" OR "); break; case ExpressionType.Equal: sb.Append(" = "); break; case ExpressionType.NotEqual: sb.Append(" != "); break; case ExpressionType.LessThan: sb.Append(" < "); break; case ExpressionType.LessThanOrEqual: sb.Append(" <= "); break; case ExpressionType.GreaterThan: sb.Append(" > "); break; case ExpressionType.GreaterThanOrEqual: sb.Append(" >= "); break; } this.Visit(b.Right); sb.Append(")"); return b; } protected override Expression VisitMethodCall(MethodCallExpression m) { string s = m.ToString(); string prop = m.Arguments[0].ToString(); sb.Append(prop.Substring(prop.IndexOf(".") + 1) + "." + m.Method.Name + "("); Visit(m.Arguments[1]); sb.Append(")"); return m; } Stack _stack = new Stack(); protected override Expression VisitMember(MemberExpression m) { var e = base.VisitMember(m); var c = m.Expression as ConstantExpression; if (c != null) { Type t = c.Value.GetType(); var x = t.InvokeMember(m.Member.Name, BindingFlags.GetField | BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, c.Value, null); if (x is string) sb.Append("\"").Append(x).Append("\""); else if (!x.GetType().IsClass) { // check for complex structs if (x is DateTime || x is Guid) sb.Append("\"").Append(x).Append("\""); else // numbers sb.Append(x); } else if (x.GetType().IsArray) { var a = x as IList; for (int i = 0; i < a.Count - 1; i++) { sb.Append(a[i]); sb.Append(","); } sb.Append(a[a.Count - 1]); } else _stack.Push(x); } if (m.Expression != null) { if (m.Expression.NodeType == ExpressionType.Parameter) // property sb.Append(m.Member.Name); else if (m.Expression.NodeType == ExpressionType.MemberAccess) // obj.property { Type t = m.Expression.Type; var f = m.Expression as MemberExpression; var val = t.InvokeMember(m.Member.Name, BindingFlags.GetField | BindingFlags.GetProperty | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, _stack.Pop(), null); if (val is string) sb.Append("\"").Append(val).Append("\""); else if (!val.GetType().IsClass) { // check for complex structs if (val is DateTime || val is Guid) sb.Append("\"").Append(val).Append("\""); else // numbers sb.Append(val); } else _stack.Push(val); } } return e; } protected override Expression VisitConstant(ConstantExpression c) { IQueryable q = c.Value as IQueryable; if (q != null) { sb.Append(q.ElementType.Name); } else if (c.Value == null) { sb.Append("NULL"); } else { //_stack.Push(c.Value); //if (Type.GetTypeCode(c.Value.GetType()) == TypeCode.Object) // _stack.Pop(); switch (Type.GetTypeCode(c.Value.GetType())) { case TypeCode.Boolean: //sb.Append(((bool)c.Value) ? 1 : 0); sb.Append(((bool)c.Value) ? "True" : "False"); break; case TypeCode.String: sb.Append("\""); sb.Append(c.Value); sb.Append("\""); break; case TypeCode.Object: break; default: sb.Append(c.Value); if (--_count > 0) sb.Append(","); break; } } return c; } int _count = 0; public override Expression Visit(Expression node) { if (node.NodeType == ExpressionType.NewArrayInit) { var a = node as NewArrayExpression; _count = a.Expressions.Count; return base.Visit(node); } else return base.Visit(node); } } } ================================================ FILE: RaptorDB.Common/Logger.cs ================================================ using System; using System.Collections.Generic; using System.Text; using System.Threading; using System.IO; using RaptorDB.Common; namespace RaptorDB { public interface ILog { /// /// Fatal log = log level 5 /// /// /// void Fatal(object msg, params object[] objs); // 5 /// /// Error log = log level 4 /// /// /// void Error(object msg, params object[] objs); // 4 /// /// Warning log = log level 3 /// /// /// void Warn(object msg, params object[] objs); // 3 /// /// Debug log = log level 2 /// /// /// void Debug(object msg, params object[] objs); // 2 /// /// Info log = log level 1 /// /// /// void Info(object msg, params object[] objs); // 1 } internal class FileLogger { // Sinlgeton pattern 4 from : http://csharpindepth.com/articles/general/singleton.aspx private static readonly FileLogger instance = new FileLogger(); // Explicit static constructor to tell C# compiler // not to mark type as beforefieldinit static FileLogger() { } private FileLogger() { } public static FileLogger Instance { get { return instance; } } private Queue _que = new Queue(); private Queue _log = new Queue(); private StreamWriter _output; private string _filename; private int _sizeLimit = 0; private long _lastSize = 0; private DateTime _lastFileDate; private bool _showMethodName = false; private string _FilePath = ""; private System.Timers.Timer _saveTimer; private int _lastLogsToKeep = 100; internal int _logabove = 1; private string _S = "" + Path.DirectorySeparatorChar; public bool ShowMethodNames { get { return _showMethodName; } } public void Init(string filename, int sizelimitKB, bool showmethodnames) { if (_output != null) return; _que = new Queue(); _showMethodName = showmethodnames; _sizeLimit = sizelimitKB; _filename = filename; // handle folder names as well -> create dir etc. _S = Path.DirectorySeparatorChar.ToString(); _FilePath = Path.GetDirectoryName(filename); if (_FilePath != "") { _FilePath = Directory.CreateDirectory(_FilePath).FullName; if (_FilePath.EndsWith(_S) == false) _FilePath += _S; } _output = new StreamWriter(filename, true); FileInfo fi = new FileInfo(filename); _lastSize = fi.Length; _lastFileDate = fi.LastWriteTime; // zip old logs ZipLogs(_FilePath, _lastFileDate); _saveTimer = new System.Timers.Timer(500); _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed); _saveTimer.Enabled = true; _saveTimer.AutoReset = true; } void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { WriteData(); } public void ShutDown() { _saveTimer.Enabled = false; WriteData(); if (_output != null) { _output.Flush(); _output.Close(); _output = null; } } private void WriteData() { if (_output == null) return; lock (_que) { while (_que.Count > 0) { object o = _que.Dequeue(); if (_output != null && o != null) { if (_sizeLimit > 0) { // implement size limited logs // implement rolling logs #region [ rolling size limit ] _lastSize += ("" + o).Length; if (_lastSize > _sizeLimit * 1000) { _output.Flush(); _output.Close(); int count = 1; while (File.Exists(_FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000"))) count++; File.Move(_filename, _FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000")); _output = new StreamWriter(_filename, true); _lastSize = 0; } #endregion } if (DateTime.Now.Subtract(_lastFileDate).Days > 0) { // implement date logs #region [ rolling dates ] _output.Flush(); _output.Close(); int count = 1; while (File.Exists(_FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000"))) { File.Move(_FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000"), _FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000") + "." + _lastFileDate.ToString("yyyy-MM-dd")); count++; } File.Move(_filename, _FilePath + Path.GetFileNameWithoutExtension(_filename) + "." + count.ToString("0000") + "." + _lastFileDate.ToString("yyyy-MM-dd")); // compress old logs here ZipLogs(_FilePath, _lastFileDate); _output = new StreamWriter(_filename, true); _lastFileDate = DateTime.Now; _lastSize = 0; #endregion } _output.Write(o); } } if (_output != null) _output.Flush(); } lock (_log) { while (_log.Count > _lastLogsToKeep) _log.Dequeue(); } } private void ZipLogs(string path, DateTime lastFileDate) { path = new DirectoryInfo(path).FullName; var prefix = path; var files = Directory.GetFiles(path, "*-*"); if (files.Length > 0) { var fn = lastFileDate.ToString("yyyy-MM-dd") + ".zip"; path += "old" + _S; if (Directory.Exists(path) == false) { fn = "0000-00-00.zip"; Directory.CreateDirectory(path); } var zip = ZipStorer.Create(path + fn, ""); foreach (var f in files) { zip.AddFile(ZipStorer.Compression.Deflate, f, f.Replace(prefix, ""), ""); File.Delete(f); } zip.Close(); } } private string FormatLog(string log, string type, string meth, string msg, object[] objs) { StringBuilder sb = new StringBuilder(); sb.Append(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); sb.Append("|"); sb.Append(log); sb.Append("|"); sb.Append(Thread.CurrentThread.ManagedThreadId.ToString()); sb.Append("|"); sb.Append(type); sb.Append("|"); sb.Append(meth); sb.Append("| "); sb.AppendLine(msg); if (objs != null) foreach (object o in objs) sb.AppendLine("" + o); return sb.ToString(); } public void Log(string logtype, string type, string meth, string msg, params object[] objs) { var l = FormatLog(logtype, type, meth, msg, objs); lock (_que) _que.Enqueue(l); lock (_log) _log.Enqueue(l); } internal List GetLastLogs() { List l = new List(); foreach (var s in _log) { l.Add(s); } return l; } public void SetLogLevel(int abovelevel) { _logabove = abovelevel; } } internal class logger : ILog { public logger(Type type) { typename = type.Namespace + "." + type.Name; } private string typename = ""; private void log(string logtype, string msg, params object[] objs) { string meth = ""; if (FileLogger.Instance.ShowMethodNames) { System.Diagnostics.StackTrace st = new System.Diagnostics.StackTrace(2); System.Diagnostics.StackFrame sf = st.GetFrame(0); meth = sf.GetMethod().Name; } FileLogger.Instance.Log(logtype, typename, meth, msg, objs); } #region ILog Members public void Fatal(object msg, params object[] objs) { log("FATAL", "" + msg, objs); } public void Error(object msg, params object[] objs) { if (FileLogger.Instance._logabove <= 4) log("ERROR", "" + msg, objs); } public void Warn(object msg, params object[] objs) { if (FileLogger.Instance._logabove <= 3) log("WARN", "" + msg, objs); } public void Debug(object msg, params object[] objs) { if (FileLogger.Instance._logabove <= 2) log("DEBUG", "" + msg, objs); } public void Info(object msg, params object[] objs) { if (FileLogger.Instance._logabove <= 1) log("INFO", "" + msg, objs); } #endregion } public static class LogManager { public static ILog GetLogger(Type obj) { return new logger(obj); } public static void Configure(string filename, int sizelimitKB, bool showmethodnames) { FileLogger.Instance.Init(filename, sizelimitKB, showmethodnames); } public static List GetLastLogs() { return FileLogger.Instance.GetLastLogs(); } public static void Shutdown() { FileLogger.Instance.ShutDown(); } public static void SetLogLevel(int abovelevel) { FileLogger.Instance.SetLogLevel(abovelevel); } } } ================================================ FILE: RaptorDB.Common/MiniLZO.cs ================================================ /** * * Modifications by Simon Hewitt * - change constructors/methods to return byte[] * - append original source size at the end of the destination buffer * - add support for MemoryStream internal buffer usage * * * ManagedLZO.MiniLZO * * Minimalistic reimplementation of minilzo in C# * * @author Shane Eric Bryldt, Copyright (C) 2006, All Rights Reserved * @note Uses unsafe/fixed pointer contexts internally * @liscence Bound by same liscence as minilzo as below, see file COPYING */ /* Based on minilzo.c -- mini subset of the LZO real-time data compression library This file is part of the LZO real-time data compression library. Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer All Rights Reserved. The LZO library is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2, as published by the Free Software Foundation. The LZO library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with the LZO library; see the file COPYING. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Markus F.X.J. Oberhumer http://www.oberhumer.com/opensource/lzo/ */ /* * NOTE: * the full LZO package can be found at * http://www.oberhumer.com/opensource/lzo/ */ using System; using System.IO; namespace RaptorDB { public class MiniLZO { private const uint M2_MAX_LEN = 8; private const uint M4_MAX_LEN = 9; private const byte M3_MARKER = 32; private const byte M4_MARKER = 16; private const uint M2_MAX_OFFSET = 0x0800; private const uint M3_MAX_OFFSET = 0x4000; private const uint M4_MAX_OFFSET = 0xbfff; private const byte BITS = 14; private const uint D_MASK = (1 << BITS) - 1; private static uint DICT_SIZE = 65536 + 3; static MiniLZO() { if (IntPtr.Size == 8) DICT_SIZE = (65536 + 3) * 2; } public static byte[] Compress(byte[] src) { return Compress(src, 0, src.Length); } public static byte[] Compress(byte[] src, int srcCount) { return Compress(src, 0, srcCount); } public static byte[] Compress(byte[] src, int srcStart, int srcLength) { byte[] workMem = new byte[DICT_SIZE]; uint dstlen = (uint)(srcLength + (srcLength / 16) + 64 + 3 + 4); byte[] dst = new byte[dstlen]; uint compressedSize = Compress(src, (uint)srcStart, (uint)srcLength, dst, 0, dstlen, workMem, 0); if (dst.Length != compressedSize) { byte[] final = new byte[compressedSize]; Buffer.BlockCopy(dst, 0, final, 0, (int)compressedSize); dst = final; } return dst; } public static byte[] Compress(MemoryStream source) { byte[] destinationBuffer; byte[] workspaceBuffer; uint sourceOffset; uint workspaceOffset; uint sourceLength; uint destinationLength; byte[] sourceBuffer = source.GetBuffer(); uint sourceCapacity = (uint)source.Capacity; sourceLength = (uint)source.Length; destinationLength = sourceLength + (sourceLength / 16) + 64 + 3 + 4; uint unusedSpace = sourceCapacity - sourceLength; uint inplaceOverhead = Math.Min(sourceLength, M4_MAX_OFFSET) + sourceLength / 64 + 16 + 3 + 4; if (unusedSpace < inplaceOverhead) { sourceOffset = 0; destinationBuffer = new byte[destinationLength]; } else { sourceOffset = inplaceOverhead; source.SetLength(sourceLength + inplaceOverhead); destinationBuffer = sourceBuffer; Buffer.BlockCopy(destinationBuffer, 0, destinationBuffer, (int)inplaceOverhead, (int)sourceLength); unusedSpace -= inplaceOverhead; } if (unusedSpace < DICT_SIZE) { workspaceBuffer = new byte[DICT_SIZE]; workspaceOffset = 0; } else { workspaceBuffer = sourceBuffer; workspaceOffset = sourceCapacity - DICT_SIZE; } uint compressedSize = Compress(sourceBuffer, sourceOffset, sourceLength, destinationBuffer, 0, destinationLength, workspaceBuffer, workspaceOffset); if (destinationBuffer == sourceBuffer) { source.SetLength(compressedSize); source.Capacity = (int)compressedSize; return source.GetBuffer(); } else { byte[] final = new byte[compressedSize]; Buffer.BlockCopy(destinationBuffer, 0, final, 0, (int)compressedSize); return final; } } private static unsafe uint Compress(byte[] src, uint srcstart, uint srcLength, byte[] dst, uint dststart, uint dstlen, byte[] workmem, uint workmemstart) { uint tmp; if (srcLength <= M2_MAX_LEN + 5) { tmp = (uint)srcLength; dstlen = 0; } else { fixed (byte* work = &workmem[workmemstart], input = &src[srcstart], output = &dst[dststart]) { byte** dict = (byte**)work; byte* in_end = input + srcLength; byte* ip_end = input + srcLength - M2_MAX_LEN - 5; byte* ii = input; byte* ip = input + 4; byte* op = output; bool literal = false; bool match = false; uint offset; uint length; uint index; byte* pos; for (; ; ) { offset = 0; index = D_INDEX1(ip); pos = ip - (ip - dict[index]); if (pos < input || (offset = (uint)(ip - pos)) <= 0 || offset > M4_MAX_OFFSET) literal = true; else if (offset <= M2_MAX_OFFSET || pos[3] == ip[3]) { } else { index = D_INDEX2(index); pos = ip - (ip - dict[index]); if (pos < input || (offset = (uint)(ip - pos)) <= 0 || offset > M4_MAX_OFFSET) literal = true; else if (offset <= M2_MAX_OFFSET || pos[3] == ip[3]) { } else literal = true; } if (!literal) { if (*((ushort*)pos) == *((ushort*)ip) && pos[2] == ip[2]) match = true; } literal = false; if (!match) { dict[index] = ip; ++ip; if (ip >= ip_end) break; continue; } match = false; dict[index] = ip; if (ip - ii > 0) { uint t = (uint)(ip - ii); if (t <= 3) { //Debug.Assert(op - 2 > output); op[-2] |= (byte)(t); } else if (t <= 18) *op++ = (byte)(t - 3); else { uint tt = t - 18; *op++ = 0; while (tt > 255) { tt -= 255; *op++ = 0; } //Debug.Assert(tt > 0); *op++ = (byte)(tt); } do { *op++ = *ii++; } while (--t > 0); } //Debug.Assert(ii == ip); ip += 3; if (pos[3] != *ip++ || pos[4] != *ip++ || pos[5] != *ip++ || pos[6] != *ip++ || pos[7] != *ip++ || pos[8] != *ip++) { --ip; length = (uint)(ip - ii); //Debug.Assert(length >= 3); //Debug.Assert(length <= M2_MAX_LEN); if (offset <= M2_MAX_OFFSET) { --offset; *op++ = (byte)(((length - 1) << 5) | ((offset & 7) << 2)); *op++ = (byte)(offset >> 3); } else if (offset <= M3_MAX_OFFSET) { --offset; *op++ = (byte)(M3_MARKER | (length - 2)); *op++ = (byte)((offset & 63) << 2); *op++ = (byte)(offset >> 6); } else { offset -= 0x4000; //Debug.Assert(offset > 0); //Debug.Assert(offset <= 0x7FFF); *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11) | (length - 2)); *op++ = (byte)((offset & 63) << 2); *op++ = (byte)(offset >> 6); } } else { byte* m = pos + M2_MAX_LEN + 1; while (ip < in_end && *m == *ip) { ++m; ++ip; } length = (uint)(ip - ii); //Debug.Assert(length > M2_MAX_LEN); if (offset <= M3_MAX_OFFSET) { --offset; if (length <= 33) *op++ = (byte)(M3_MARKER | (length - 2)); else { length -= 33; *op++ = M3_MARKER | 0; while (length > 255) { length -= 255; *op++ = 0; } //Debug.Assert(length > 0); *op++ = (byte)(length); } } else { offset -= 0x4000; //Debug.Assert(offset > 0); //Debug.Assert(offset <= 0x7FFF); if (length <= M4_MAX_LEN) *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11) | (length - 2)); else { length -= M4_MAX_LEN; *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11)); while (length > 255) { length -= 255; *op++ = 0; } //Debug.Assert(length > 0); *op++ = (byte)(length); } } *op++ = (byte)((offset & 63) << 2); *op++ = (byte)(offset >> 6); } ii = ip; if (ip >= ip_end) break; } dstlen = (uint)(op - output); tmp = (uint)(in_end - ii); } } if (tmp > 0) { uint ii = (uint)srcLength - tmp + srcstart; if (dstlen == 0 && tmp <= 238) { dst[dstlen++] = (byte)(17 + tmp); } else if (tmp <= 3) { dst[dstlen - 2] |= (byte)(tmp); } else if (tmp <= 18) { dst[dstlen++] = (byte)(tmp - 3); } else { uint tt = tmp - 18; dst[dstlen++] = 0; while (tt > 255) { tt -= 255; dst[dstlen++] = 0; } //Debug.Assert(tt > 0); dst[dstlen++] = (byte)(tt); } do { dst[dstlen++] = src[ii++]; } while (--tmp > 0); } dst[dstlen++] = M4_MARKER | 1; dst[dstlen++] = 0; dst[dstlen++] = 0; // Append the source count dst[dstlen++] = (byte)srcLength; dst[dstlen++] = (byte)(srcLength >> 8); dst[dstlen++] = (byte)(srcLength >> 16); dst[dstlen++] = (byte)(srcLength >> 24); return dstlen; } public static unsafe byte[] Decompress(byte[] src) { byte[] dst = new byte[(src[src.Length - 4] | (src[src.Length - 3] << 8) | (src[src.Length - 2] << 16 | src[src.Length - 1] << 24))]; uint t = 0; fixed (byte* input = src, output = dst) { byte* pos = null; byte* ip_end = input + src.Length - 4; byte* op_end = output + dst.Length; byte* ip = input; byte* op = output; bool match = false; bool match_next = false; bool match_done = false; bool copy_match = false; bool first_literal_run = false; bool eof_found = false; if (*ip > 17) { t = (uint)(*ip++ - 17); if (t < 4) match_next = true; else { //Debug.Assert(t > 0); if ((op_end - op) < t) throw new OverflowException("Output Overrun"); if ((ip_end - ip) < t + 1) throw new OverflowException("Input Overrun"); do { *op++ = *ip++; } while (--t > 0); first_literal_run = true; } } while (!eof_found && ip < ip_end) { if (!match_next && !first_literal_run) { t = *ip++; if (t >= 16) match = true; else { if (t == 0) { if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); while (*ip == 0) { t += 255; ++ip; if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); } t += (uint)(15 + *ip++); } //Debug.Assert(t > 0); if ((op_end - op) < t + 3) throw new OverflowException("Output Overrun"); if ((ip_end - ip) < t + 4) throw new OverflowException("Input Overrun"); for (int x = 0; x < 4; ++x, ++op, ++ip) *op = *ip; if (--t > 0) { if (t >= 4) { do { for (int x = 0; x < 4; ++x, ++op, ++ip) *op = *ip; t -= 4; } while (t >= 4); if (t > 0) { do { *op++ = *ip++; } while (--t > 0); } } else { do { *op++ = *ip++; } while (--t > 0); } } } } if (!match && !match_next) { first_literal_run = false; t = *ip++; if (t >= 16) match = true; else { pos = op - (1 + M2_MAX_OFFSET); pos -= t >> 2; pos -= *ip++ << 2; if (pos < output || pos >= op) throw new OverflowException("Lookbehind Overrun"); if ((op_end - op) < 3) throw new OverflowException("Output Overrun"); *op++ = *pos++; *op++ = *pos++; *op++ = *pos++; match_done = true; } } match = false; do { if (t >= 64) { pos = op - 1; pos -= (t >> 2) & 7; pos -= *ip++ << 3; t = (t >> 5) - 1; if (pos < output || pos >= op) throw new OverflowException("Lookbehind Overrun"); if ((op_end - op) < t + 2) throw new OverflowException("Output Overrun"); copy_match = true; } else if (t >= 32) { t &= 31; if (t == 0) { if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); while (*ip == 0) { t += 255; ++ip; if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); } t += (uint)(31 + *ip++); } pos = op - 1; pos -= (*(ushort*)ip) >> 2; ip += 2; } else if (t >= 16) { pos = op; pos -= (t & 8) << 11; t &= 7; if (t == 0) { if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); while (*ip == 0) { t += 255; ++ip; if ((ip_end - ip) < 1) throw new OverflowException("Input Overrun"); } t += (uint)(7 + *ip++); } pos -= (*(ushort*)ip) >> 2; ip += 2; if (pos == op) eof_found = true; else pos -= 0x4000; } else { pos = op - 1; pos -= t >> 2; pos -= *ip++ << 2; if (pos < output || pos >= op) throw new OverflowException("Lookbehind Overrun"); if ((op_end - op) < 2) throw new OverflowException("Output Overrun"); *op++ = *pos++; *op++ = *pos++; match_done = true; } if (!eof_found && !match_done && !copy_match) { if (pos < output || pos >= op) throw new OverflowException("Lookbehind Overrun"); //Debug.Assert(t > 0); if ((op_end - op) < t + 2) throw new OverflowException("Output Overrun"); } if (!eof_found && t >= 2 * 4 - 2 && (op - pos) >= 4 && !match_done && !copy_match) { for (int x = 0; x < 4; ++x, ++op, ++pos) *op = *pos; t -= 2; do { for (int x = 0; x < 4; ++x, ++op, ++pos) *op = *pos; t -= 4; } while (t >= 4); if (t > 0) { do { *op++ = *pos++; } while (--t > 0); } } else if (!eof_found && !match_done) { copy_match = false; *op++ = *pos++; *op++ = *pos++; do { *op++ = *pos++; } while (--t > 0); } if (!eof_found && !match_next) { match_done = false; t = (uint)(ip[-2] & 3); if (t == 0) break; } if (!eof_found) { match_next = false; //Debug.Assert(t > 0); //Debug.Assert(t < 4); if ((op_end - op) < t) throw new OverflowException("Output Overrun"); if ((ip_end - ip) < t + 1) throw new OverflowException("Input Overrun"); *op++ = *ip++; if (t > 1) { *op++ = *ip++; if (t > 2) *op++ = *ip++; } t = *ip++; } } while (!eof_found && ip < ip_end); } if (!eof_found) throw new OverflowException("EOF Marker Not Found"); else { //Debug.Assert(t == 1); if (ip > ip_end) throw new OverflowException("Input Overrun"); else if (ip < ip_end) throw new OverflowException("Input Not Consumed"); } } return dst; } private unsafe static uint D_INDEX1(byte* input) { return D_MS(D_MUL(0x21, D_X3(input, 5, 5, 6)) >> 5, 0); } private static uint D_INDEX2(uint idx) { return (idx & (D_MASK & 0x7FF)) ^ (((D_MASK >> 1) + 1) | 0x1F); } private static uint D_MS(uint v, byte s) { return (v & (D_MASK >> s)) << s; } private static uint D_MUL(uint a, uint b) { return a * b; } private unsafe static uint D_X2(byte* input, byte s1, byte s2) { return (uint)((((input[2] << s2) ^ input[1]) << s1) ^ input[0]); } private unsafe static uint D_X3(byte* input, byte s1, byte s2, byte s3) { return (D_X2(input + 1, s2, s3) << s1) ^ input[0]; } } } ================================================ FILE: RaptorDB.Common/MurMurHash2.cs ================================================ using System; namespace RaptorDB.Common { //internal static class murmur3 //{ // private static uint seed = 7878; // public static uint MurmurHash3(byte[] data) // { // const uint c1 = 0xcc9e2d51; // const uint c2 = 0x1b873593; // int curLength = data.Length; /* Current position in byte array */ // int length = curLength; /* the const length we need to fix tail */ // uint h1 = seed; // uint k1 = 0; // /* body, eat stream a 32-bit int at a time */ // int currentIndex = 0; // while (curLength >= 4) // { // /* Get four bytes from the input into an UInt32 */ // k1 = (uint)(data[currentIndex++] // | data[currentIndex++] << 8 // | data[currentIndex++] << 16 // | data[currentIndex++] << 24); // /* bitmagic hash */ // k1 *= c1; // k1 = rotl32(k1, 15); // k1 *= c2; // h1 ^= k1; // h1 = rotl32(h1, 13); // h1 = h1 * 5 + 0xe6546b64; // curLength -= 4; // } // /* tail, the reminder bytes that did not make it to a full int */ // /* (this switch is slightly more ugly than the C++ implementation // * because we can't fall through) */ // switch (curLength) // { // case 3: // k1 = (UInt32)(data[currentIndex++] // | data[currentIndex++] << 8 // | data[currentIndex++] << 16); // k1 *= c1; // k1 = rotl32(k1, 15); // k1 *= c2; // h1 ^= k1; // break; // case 2: // k1 = (UInt32)(data[currentIndex++] // | data[currentIndex++] << 8); // k1 *= c1; // k1 = rotl32(k1, 15); // k1 *= c2; // h1 ^= k1; // break; // case 1: // k1 = (UInt32)(data[currentIndex++]); // k1 *= c1; // k1 = rotl32(k1, 15); // k1 *= c2; // h1 ^= k1; // break; // }; // // finalization, magic chants to wrap it all up // h1 ^= (uint)length; // h1 = fmix(h1); // unchecked // { // return (uint)h1; // } // } // private static uint rotl32(uint x, byte r) // { // return (x << r) | (x >> (32 - r)); // } // private static uint fmix(uint h) // { // h ^= h >> 16; // h *= 0x85ebca6b; // h ^= h >> 13; // h *= 0xc2b2ae35; // h ^= h >> 16; // return h; // } //} public class MurmurHash2Unsafe { public UInt32 Hash(Byte[] data) { return Hash(data, 0xc58f1a7b); } const UInt32 m = 0x5bd1e995; const Int32 r = 24; public unsafe UInt32 Hash(Byte[] data, UInt32 seed) { Int32 length = data.Length; if (length == 0) return 0; UInt32 h = seed ^ (UInt32)length; Int32 remainingBytes = length & 3; // mod 4 Int32 numberOfLoops = length >> 2; // div 4 fixed (byte* firstByte = &(data[0])) { UInt32* realData = (UInt32*)firstByte; while (numberOfLoops != 0) { UInt32 k = *realData; k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; numberOfLoops--; realData++; } switch (remainingBytes) { case 3: h ^= (UInt16)(*realData); h ^= ((UInt32)(*(((Byte*)(realData)) + 2))) << 16; h *= m; break; case 2: h ^= (UInt16)(*realData); h *= m; break; case 1: h ^= *((Byte*)realData); h *= m; break; default: break; } } // Do a few final mixes of the hash to ensure the last few // bytes are well-incorporated. h ^= h >> 13; h *= m; h ^= h >> 15; return h; } } } ================================================ FILE: RaptorDB.Common/NetworkClient.cs ================================================ using System; using System.Net.Sockets; using System.Threading; using System.Net; using System.Threading.Tasks; namespace RaptorDB.Common { // // Header bits format : 0 - json = 1 , bin = 0 // 1 - binaryjson = 1 , text json = 0 // 2 - compressed = 1 , uncompressed = 0 // // 0 : data format // 1-4 : data length public class NetworkClient { public static class Config { /// /// Block buffer size (default = 32kb) /// public static int BufferSize = 32 * 1024; /// /// Log data if over (default = 1,000,000) /// public static int LogDataSizesOver = 1000000; /// /// Compress data if over (default = 1,000,000) /// public static int CompressDataOver = 1000000; /// /// Kill inactive client connections (default = 30sec) /// public static int KillConnectionSeconds = 30; } public NetworkClient(string server, int port) { _server = server; _port = port; } private ILog log = LogManager.GetLogger(typeof(NetworkClient)); private TcpClient _client; private string _server; private int _port; public Guid ClientID = Guid.NewGuid(); public bool UseBJSON = true; public void Connect() { _client = new TcpClient(_server, _port); _client.SendBufferSize = Config.BufferSize; _client.ReceiveBufferSize = _client.SendBufferSize; } public object Send(object data) { try { CheckConnection(); byte[] hdr = new byte[5]; hdr[0] = (UseBJSON ? (byte)3 : (byte)0); byte[] dat = fastBinaryJSON.BJSON.ToBJSON(data); bool compressed = false; if (dat.Length > NetworkClient.Config.CompressDataOver) { log.Debug("compressing data over limit : " + dat.Length.ToString("#,#")); compressed = true; dat = MiniLZO.Compress(dat); log.Debug("new size : " + dat.Length.ToString("#,#")); } byte[] len = Helper.GetBytes(dat.Length, false); hdr[0] = (byte)(3 + (compressed ? 4 : 0)); Array.Copy(len, 0, hdr, 1, 4); _client.Client.Send(hdr); _client.Client.Send(dat); byte[] rechdr = new byte[5]; using (NetworkStream n = new NetworkStream(_client.Client)) { n.Read(rechdr, 0, 5); int c = Helper.ToInt32(rechdr, 1); byte[] recd = new byte[c]; int bytesRead = 0; int chunksize = 1; while (bytesRead < c && chunksize > 0) bytesRead += chunksize = n.Read (recd, bytesRead, c - bytesRead); if ((rechdr[0] & (byte)4) == (byte)4) recd = MiniLZO.Decompress(recd); if ((rechdr[0] & (byte)3) == (byte)3) return fastBinaryJSON.BJSON.ToObject(recd); } } catch { } return null; } private void CheckConnection() { // check connected state before sending if (_client == null || !_client.Connected) Connect(); } public void Close() { if (_client != null) { _client.Close(); } } } public class NetworkServer { public delegate object ProcessPayload(object data); private ILog log = LogManager.GetLogger(typeof(NetworkServer)); ProcessPayload _handler; private bool _run = true; private int count = 0; private int _port; public void Start(int port, ProcessPayload handler) { _handler = handler; _port = port; ThreadPool.SetMinThreads(50, 50); System.Timers.Timer t = new System.Timers.Timer(1000); t.AutoReset = true; t.Start(); t.Elapsed += new System.Timers.ElapsedEventHandler(t_Elapsed); Task.Factory.StartNew(() => Run(), TaskCreationOptions.AttachedToParent); } private void Run() { TcpListener listener = new TcpListener(IPAddress.Any, _port); listener.Start(); while (_run) { try { TcpClient c = listener.AcceptTcpClient(); Task.Factory.StartNew(() => Accept(c)); } catch (Exception ex) { log.Error(ex); } } } void t_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { if (count > 0) log.Info("tcp connects/sec = " + count); count = 0; } public void Stop() { _run = false; } void Accept(TcpClient client) { using (NetworkStream n = client.GetStream()) { while (client.Connected) { this.count++; byte[] c = new byte[5]; n.Read(c, 0, 5); int count = BitConverter.ToInt32(c, 1); byte[] data = new byte[count]; int bytesRead = 0; int chunksize = 1; while (bytesRead < count && chunksize > 0) bytesRead += chunksize = n.Read (data, bytesRead, count - bytesRead); object o = fastBinaryJSON.BJSON.ToObject(data); if ((c[0] & (byte)4) == (byte)4) data = MiniLZO.Decompress(data); object r = _handler(o); bool compressed = false; var dataret = fastBinaryJSON.BJSON.ToBJSON(r); r = null; if (dataret.Length > NetworkClient.Config.CompressDataOver) { log.Debug("compressing data over limit : " + dataret.Length.ToString("#,#")); compressed = true; dataret = MiniLZO.Compress(dataret); log.Debug("new size : " + dataret.Length.ToString("#,#")); } if (dataret.Length > NetworkClient.Config.LogDataSizesOver) log.Debug("data size (bytes) = " + dataret.Length.ToString("#,#")); byte[] b = BitConverter.GetBytes(dataret.Length); byte[] hdr = new byte[5]; hdr[0] = (byte)(3 + (compressed ? 4 : 0)); Array.Copy(b, 0, hdr, 1, 4); n.Write(hdr, 0, 5); n.Write(dataret, 0, dataret.Length); //n.Flush(); //return; int wait = 0; bool close = false; var dt = FastDateTime.Now; while (n.DataAvailable == false && close == false) { wait++; if (wait < 10000) // kludge : for insert performance Thread.Sleep(0); else { Thread.Sleep(1); // wait done -> close connection if (FastDateTime.Now.Subtract(dt).TotalSeconds > NetworkClient.Config.KillConnectionSeconds) close = true; } } if (close) break; } n.Close(); } client.Close(); } } } ================================================ FILE: RaptorDB.Common/Packets.cs ================================================ using System; namespace RaptorDB.Common { public class Packet { public Packet() { OrderBy = ""; } public string Username { get; set; } public string PasswordHash { get; set; } //public int Token { get; set; } //public int Session { get; set; } public string Command { get; set; } public object Data { get; set; } public Guid Docid { get; set; } public string Viewname { get; set; } public int Start { get; set; } public int Count { get; set; } public string OrderBy { get; set; } public Guid ClientID { get; set; } public string InstanceName { get; set; } } public class ReturnPacket { public ReturnPacket() { } public ReturnPacket(bool ok) { OK = ok; } public ReturnPacket(bool ok, string err) { OK = ok; Error = err; } public string Error { get; set; } public bool OK { get; set; } //public int Token { get; set; } //public int Session { get; set; } public object Data { get; set; } } } ================================================ FILE: RaptorDB.Common/Properties/AssemblyInfo.cs ================================================ using System.Reflection; using System.Security; [assembly: AssemblyTitle("RaptorDB.Common")] [assembly: AssemblyDescription("Common classes for RaptorDB client and server")] [assembly: AssemblyProduct("RaptorDB.Common")] ================================================ FILE: RaptorDB.Common/RaptorDB.Common.csproj ================================================  Debug AnyCPU 8.0.30703 2.0 {32331D51-5BE0-41E2-AF1A-9B086C5AE809} Library Properties RaptorDB.Common RaptorDB.Common 512 True full true bin\Debug\ TRACE;DEBUG;NETSERVER; NETJSON; net4;NET4 prompt 4 True false pdbonly True bin\Release\ TRACE;NETSERVER; NETJSON; net4 prompt 4 false true ..\raptordb.snk true BuildVersion.cs md "$(SolutionDir)output\net40" copy "$(TargetPath)" "$(SolutionDir)output\net40\$(TargetFileName)" "$(SolutionDir)tools\buildversion.exe" "$(SolutionDir)buildversion.cs" ================================================ FILE: RaptorDB.Common/RaptorDBClient.cs ================================================ using System; using System.Linq; using System.Text; using RaptorDB.Common; using System.Linq.Expressions; using System.Reflection; namespace RaptorDB { public class KVHF : IKeyStoreHF { public KVHF(NetworkClient client, string username, string password) { _client = client; _username = username; _password = password; } NetworkClient _client; private string _username; private string _password; public object GetObjectHF(string key) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.GetObjectHF; p.Data = key; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) return ret.Data; else return null; } public bool SetObjectHF(string key, object obj) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.SetObjectHF; p.Data = new object[] { key, obj }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } public bool DeleteKeyHF(string key) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DeleteKeyHF; p.Data = key; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (bool)ret.Data; } public int CountHF() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.CountHF; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } public bool ContainsHF(string key) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ContainsHF; p.Data = key; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (bool)ret.Data; } public string[] GetKeysHF() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.GetKeysHF; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ((object[])ret.Data).Cast().ToArray(); } public void CompactStorageHF() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.CompactStorageHF; ReturnPacket ret = (ReturnPacket)_client.Send(p); return; } private Packet CreatePacket() { Packet p = new Packet(); p.Username = _username; p.PasswordHash = Helper.MurMur.Hash(Encoding.UTF8.GetBytes(_username + "|" + _password)).ToString(); p.ClientID = _client.ClientID; return p; } public int Increment(string key, int amount) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.IncrementHF; p.Data = new object[] { key, amount }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } public int Decrement(string key, int amount) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DecrementHF; p.Data = new object[] { key, amount }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } public decimal Increment(string key, decimal amount) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.IncrementHF; p.Data = new object[] { key, amount }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (decimal)ret.Data; } public decimal Decrement(string key, decimal amount) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DecrementHF; p.Data = new object[] { key, amount }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (decimal)ret.Data; } } public class RaptorDBClient : IRaptorDB { public RaptorDBClient(string server, int port, string username, string password) { _username = username; _password = password; _client = new NetworkClient(server, port); // speed settings fastJSON.JSON.Parameters.ParametricConstructorOverride = true; fastBinaryJSON.BJSON.Parameters.ParametricConstructorOverride = true; _kv = new KVHF(_client, _username, _password); } private KVHF _kv; private NetworkClient _client; private string _username; private string _password; private SafeDictionary _assembly = new SafeDictionary(); /// /// Save a document to RaptorDB /// /// /// /// /// public bool Save(Guid docID, T document) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Save; p.Docid = docID; p.Data = document; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Save a file to RaptorDB /// /// /// /// public bool SaveBytes(Guid fileID, byte[] bytes) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.SaveBytes; p.Docid = fileID; p.Data = bytes; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Query any view -> get all rows /// /// /// /// public Result Query(string viewname) { return Query(viewname, 0, -1); } /// /// Query a view using a string filter /// /// /// /// public Result Query(string viewname, string filter) { return Query(viewname, filter, 0, -1); } /// /// Fetch a document by it's ID /// /// /// public object Fetch(Guid docID) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Fetch; p.Docid = docID; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) return ret.Data; else return null; } /// /// Fetch file data by it's ID /// /// /// public byte[] FetchBytes(Guid fileID) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FetchBytes; p.Docid = fileID; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) return (byte[])ret.Data; else return null; } /// /// Shutdown and cleanup /// public void Shutdown() { try { // send close packet Packet p = CreatePacket(); p.Command = "_close"; ReturnPacket ret = (ReturnPacket)_client.Send(p); _client.Close(); } catch { } } /// /// Backup the data file in incremental mode to the RaptorDB folder /// /// public bool Backup() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Backup; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Restore backup files stored in RaptorDB folder /// public void Restore() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Restore; ReturnPacket ret = (ReturnPacket)_client.Send(p); } /// /// Delete a document (the actual data is not deleted just marked so) /// /// /// public bool Delete(Guid docid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Delete; p.Docid = docid; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Delete a file (the actual data is not deleted just marked so) /// /// /// public bool DeleteBytes(Guid fileid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DeleteBytes; p.Docid = fileid; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Add a user for server mode login /// /// /// /// /// public bool AddUser(string username, string oldpassword, string newpassword) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.AddUser; p.Data = new object[] { username, oldpassword, newpassword }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.OK; } /// /// Execute server side queries /// /// /// /// public object[] ServerSide(ServerSideFunc func, string filter) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ServerSide; p.Data = new object[] { func.Method.ReflectedType.AssemblyQualifiedName, func.Method.Name, filter }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (object[])ret.Data; } /// /// Execute server side queries /// /// /// /// /// public object[] ServerSide(ServerSideFunc func, Expression> filter) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.ServerSide; p.Data = new object[] { func.Method.ReflectedType.AssemblyQualifiedName, func.Method.Name, ls.sb.ToString() }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (object[])ret.Data; } /// /// Full text search the complete original document /// /// /// public int[] FullTextSearch(string filter) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FullText; p.Data = new object[] { filter }; ReturnPacket ret = (ReturnPacket)_client.Send(p); object[] r = (object[])ret.Data; return Array.ConvertAll(r, i => (int)i); } private Packet CreatePacket() { Packet p = new Packet(); p.Username = _username; p.PasswordHash = Helper.MurMur.Hash(Encoding.UTF8.GetBytes(_username + "|" + _password)).ToString(); p.ClientID = _client.ClientID; return p; } /// /// Query all data in a view with paging /// /// /// /// /// public Result Query(string viewname, int start, int count) { return Query(viewname, "", start, count); } /// /// Query a View with a string filter with paging /// /// /// /// /// /// public Result Query(string viewname, string filter, int start, int count, string orderby) { bool b = false; // check if return type exists and copy assembly if needed if (_assembly.TryGetValue(viewname, out b) == false) { Packet pp = CreatePacket(); pp.Command = "" + COMMANDS.CheckAssembly; pp.Viewname = viewname; ReturnPacket r = (ReturnPacket)_client.Send(pp); string type = r.Error; Type t = Type.GetType(type); if (t == null) { if (r.Data != null) { var a = Assembly.Load((byte[])r.Data); _assembly.Add(viewname, true); } } else _assembly.Add(viewname, true); } Packet p = CreatePacket(); p.Command = "" + COMMANDS.QueryStr; p.Viewname = viewname; p.Data = filter; p.Start = start; p.Count = count; p.OrderBy = orderby; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (Result)ret.Data; } /// /// Query a View with a LINQ filter with paging /// /// /// /// /// /// /// public Result Query(string viewname, Expression> filter, int start, int count, string orderby) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.QueryStr; p.Viewname = viewname; p.Start = start; p.Count = count; p.Data = ls.sb.ToString(); p.OrderBy = orderby; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (Result)ret.Data; } /// /// Count rows /// /// /// public int Count(string viewname) { return Count(viewname, ""); } /// /// Count rows with a string filter /// /// /// /// public int Count(string viewname, string filter) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.CountStr; p.Viewname = viewname; p.Data = filter; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } /// /// Query with LINQ filter /// /// /// /// public Result Query(Expression> filter) { return Query(filter, 0, -1, ""); } /// /// Query with LINQ filter and paging /// /// /// /// /// /// public Result Query(Expression> filter, int start, int count, string orderby) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.QueryType; p.Start = start; p.Count = count; p.OrderBy = orderby; p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, ls.sb.ToString() }; ReturnPacket ret = (ReturnPacket)_client.Send(p); Result res = (Result)ret.Data; return GenericResult(res); } private static Result GenericResult(Result res) { // FEATURE : dirty hack here to cleanup Result result = new Result(); if (res != null) { result.Count = res.Count; result.EX = res.EX; result.OK = res.OK; result.TotalCount = res.TotalCount; if (res.Rows != null) result.Rows = res.Rows.Cast().ToList(); else result.Rows = null; } return result; } /// /// Query with string filter /// /// /// /// public Result Query(string filter) { return Query(filter, 0, -1, ""); } /// /// Query with string filter and paging /// /// /// /// /// /// public Result Query(string filter, int start, int count, string orderby) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.QueryType; p.Start = start; p.Count = count; p.OrderBy = orderby; p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, filter }; ReturnPacket ret = (ReturnPacket)_client.Send(p); Result res = (Result)ret.Data; return GenericResult(res); } /// /// Count with LINQ filter /// /// /// /// public int Count(Expression> filter) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.GCount; p.Viewname = typeof(TRowSchema).AssemblyQualifiedName; p.Data = ls.sb.ToString(); ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } /// /// Fetch the document change history /// /// /// public int[] FetchHistory(Guid docid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DocHistory; p.Docid = docid; ReturnPacket ret = (ReturnPacket)_client.Send(p); object[] r = (object[])ret.Data; return Array.ConvertAll(r, i => (int)i); } /// /// Fetch the file change history /// /// /// public int[] FetchBytesHistory(Guid fileid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FileHistory; p.Docid = fileid; ReturnPacket ret = (ReturnPacket)_client.Send(p); object[] r = (object[])ret.Data; return Array.ConvertAll(r, i => (int)i); } /// /// Fetch a specific document version /// /// /// public object FetchVersion(int versionNumber) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FetchVersion; p.Data = versionNumber; ReturnPacket ret = (ReturnPacket)_client.Send(p); return ret.Data; } /// /// Fetch a specific file version /// /// /// public byte[] FetchBytesVersion(int versionNumber) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FetchFileVersion; p.Data = versionNumber; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (byte[])ret.Data; } /// /// Query a View with a string filter with paging /// /// /// /// /// /// public Result Query(string viewname, string filter, int start, int count) { return this.Query(viewname, filter, start, count, ""); } /// /// Query a view with paging /// /// /// /// /// /// /// public Result Query(string viewname, Expression> filter, int start, int count) { return this.Query(viewname, filter, start, count, ""); } /// /// Query a view with paging /// /// /// /// /// /// public Result Query(Expression> filter, int start, int count) { return Query(filter, start, count, ""); } /// /// Query a view with paging /// /// /// /// /// /// public Result Query(string filter, int start, int count) { return Query(filter, start, count, ""); } /// /// Fetch a change history for a document with dates /// /// /// public HistoryInfo[] FetchHistoryInfo(Guid docid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FetchHistoryInfo; p.Docid = docid; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (HistoryInfo[])ret.Data; } /// /// Fetch a change history for a file with dates /// /// /// public HistoryInfo[] FetchBytesHistoryInfo(Guid docid) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FetchByteHistoryInfo; p.Docid = docid; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (HistoryInfo[])ret.Data; } /// /// Delete directly from a view using a filter /// /// /// /// public int ViewDelete(Expression> filter) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.ViewDelete_t; p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, ls.sb.ToString() }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } /// /// Delete directly from a view using a filter /// /// /// /// public int ViewDelete(string viewname, string filter) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ViewDelete; p.Data = new object[] { viewname, filter }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (int)ret.Data; } /// /// Insert directly into a view /// /// /// /// /// public bool ViewInsert(Guid id, TRowSchema row) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ViewInsert_t; p.Docid = id; p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, row }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (bool)ret.Data; } /// /// Insert directly into a view /// /// /// /// /// public bool ViewInsert(string viewname, Guid id, object row) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ViewInsert; p.Docid = id; p.Data = new object[] { viewname, row }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (bool)ret.Data; } /// /// Get the number of documents in the storage file regardless of versions /// /// public long DocumentCount() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.DocCount; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (long)ret.Data; } public IKeyStoreHF GetKVHF() { return _kv; } public object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args) { Packet p = CreatePacket(); p.Command = "" + COMMANDS.ServerSideWithArgs; p.Data = new object[] { func.Method.ReflectedType.AssemblyQualifiedName, func.Method.Name, filter , args }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (object[])ret.Data; } public object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args) { LINQString ls = new LINQString(); ls.Visit(filter); Packet p = CreatePacket(); p.Command = "" + COMMANDS.ServerSideWithArgs; p.Data = new object[] { func.Method.ReflectedType.AssemblyQualifiedName, func.Method.Name, ls.sb.ToString(), args }; ReturnPacket ret = (ReturnPacket)_client.Send(p); return (object[])ret.Data; } public T Fetch(Guid docID) where T : class { Packet p = CreatePacket(); p.Command = "" + COMMANDS.Fetch; p.Docid = docID; ReturnPacket ret = (ReturnPacket)_client.Send(p); if (ret.OK) return (ret.Data as T); else return null; } public void FreeMemory() { Packet p = CreatePacket(); p.Command = "" + COMMANDS.FreeMemory; ReturnPacket ret = (ReturnPacket)_client.Send(p); } public void SaveToDocsOnViewInsert(bool yes) { // FIX: set on server } } } ================================================ FILE: RaptorDB.Common/SafeDictionary.cs ================================================ using System; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; namespace RaptorDB.Common { public interface IKV { bool TryGetValue(T key, out V val); int Count(); IEnumerator> GetEnumerator(); void Add(T key, V value); T[] Keys(); bool Remove(T key); void Clear(); V GetValue(T key); // safesortedlist only //V GetValue(int index); //T GetKey(int index); } public class ReferenceEqualityComparer : IEqualityComparer, IEqualityComparer { public static ReferenceEqualityComparer Default { get; } = new ReferenceEqualityComparer(); public new bool Equals(object x, object y) => x.Equals(y); public int GetHashCode(object obj) => RuntimeHelpers.GetHashCode(obj); } public class SafeDictionary : IKV { private readonly object _Padlock = new object(); private readonly Dictionary _Dictionary; public SafeDictionary(int capacity) { _Dictionary = new Dictionary(capacity); } public SafeDictionary() { _Dictionary = new Dictionary(); } public bool TryGetValue(TKey key, out TValue value) { lock (_Padlock) return _Dictionary.TryGetValue(key, out value); } public TValue this[TKey key] { get { lock (_Padlock) return _Dictionary[key]; } set { lock (_Padlock) _Dictionary[key] = value; } } public int Count() { lock (_Padlock) return _Dictionary.Count; } public IEnumerator> GetEnumerator() { return ((ICollection>)_Dictionary).GetEnumerator(); } public void Add(TKey key, TValue value) { lock (_Padlock) { if (_Dictionary.ContainsKey(key) == false) _Dictionary.Add(key, value); else _Dictionary[key] = value; } } public TKey[] Keys() { lock (_Padlock) { TKey[] keys = new TKey[_Dictionary.Keys.Count]; _Dictionary.Keys.CopyTo(keys, 0); return keys; } } public bool Remove(TKey key) { if (key == null) return true; lock (_Padlock) { return _Dictionary.Remove(key); } } public void Clear() { lock (_Padlock) _Dictionary.Clear(); } public TValue GetValue(TKey key) { lock (_Padlock) return _Dictionary[key]; } } public class SafeSortedList : IKV { private object _padlock = new object(); SortedList _list = new SortedList(); public int Count() { lock (_padlock) return _list.Count; } public void Add(T key, V val) { lock (_padlock) { if (_list.ContainsKey(key) == false) _list.Add(key, val); else _list[key] = val; } } public bool Remove(T key) { if (key == null) return true; lock (_padlock) return _list.Remove(key); } public T GetKey(int index) { lock (_padlock) if (index < _list.Count) return _list.Keys[index]; else return default(T); } public V GetValue(int index) { lock (_padlock) if (index < _list.Count) return _list.Values[index]; else return default(V); } public T[] Keys() { lock (_padlock) { T[] keys = new T[_list.Keys.Count]; _list.Keys.CopyTo(keys, 0); return keys; } } public V this[T key] { get { lock (_padlock) return _list[key]; } set { lock (_padlock) _list[key] = value; } } public IEnumerator> GetEnumerator() { return ((ICollection>)_list).GetEnumerator(); } public bool TryGetValue(T key, out V value) { lock (_padlock) return _list.TryGetValue(key, out value); } public void Clear() { lock (_padlock) _list.Clear(); } public V GetValue(T key) { lock (_padlock) return _list[key]; } } //------------------------------------------------------------------------------------------------------------------ public static class FastDateTime { public static TimeSpan LocalUtcOffset; public static DateTime Now { get { return DateTime.SpecifyKind(DateTime.UtcNow + LocalUtcOffset, DateTimeKind.Local); } } static FastDateTime() { LocalUtcOffset = TimeZone.CurrentTimeZone.GetUtcOffset(DateTime.Now); } } //------------------------------------------------------------------------------------------------------------------ public static class Helper { public static MurmurHash2Unsafe MurMur = new MurmurHash2Unsafe(); public static int CompareMemCmp(byte[] left, byte[] right) { int c = left.Length; if (c > right.Length) c = right.Length; return memcmp(left, right, c); } [DllImport("msvcrt.dll", CallingConvention = CallingConvention.Cdecl)] private static extern int memcmp(byte[] arr1, byte[] arr2, int cnt); public static int ToInt32(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[4]; Buffer.BlockCopy(value, startIndex, b, 0, 4); Array.Reverse(b); return ToInt32(b, 0); } return ToInt32(value, startIndex); } public static unsafe int ToInt32(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *((int*)numRef); } } public static long ToInt64(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[8]; Buffer.BlockCopy(value, startIndex, b, 0, 8); Array.Reverse(b); return ToInt64(b, 0); } return ToInt64(value, startIndex); } public static unsafe long ToInt64(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((long*)numRef)); } } public static short ToInt16(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[2]; Buffer.BlockCopy(value, startIndex, b, 0, 2); Array.Reverse(b); return ToInt16(b, 0); } return ToInt16(value, startIndex); } public static unsafe short ToInt16(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((short*)numRef)); } } public static unsafe byte[] GetBytes(long num, bool reverse) { byte[] buffer = new byte[8]; fixed (byte* numRef = buffer) { *((long*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } public static unsafe byte[] GetBytes(int num, bool reverse) { byte[] buffer = new byte[4]; fixed (byte* numRef = buffer) { *((int*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } public static unsafe byte[] GetBytes(short num, bool reverse) { byte[] buffer = new byte[2]; fixed (byte* numRef = buffer) { *((short*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } public static byte[] GetBytes(string s) { return Encoding.UTF8.GetBytes(s); // TODO : change to unicode ?? } public static string GetString(byte[] buffer, int index, short length) { return Encoding.UTF8.GetString(buffer, index, length); // TODO : change to unicode ?? } } } ================================================ FILE: RaptorDB.Common/View.cs ================================================ using System; using System.Collections.Generic; using System.Xml.Serialization; namespace RaptorDB { public abstract class ViewBase { public delegate void MapFunctionDelgate(IMapAPI api, Guid docid, V doc); /// /// Increment this when you change view definitions so the engine can rebuild the contents /// public int Version { get; set; } /// /// Name of the view will be used for foldernames and filename and generated code /// public string Name { get; set;} /// /// A text for describing this views purpose for other developers /// public string Description { get; set; } /// /// Column definitions for the view storage /// [XmlIgnore] public Type Schema { get; set; } /// /// Is this the primary list and will be populated synchronously /// public bool isPrimaryList { get; set; } /// /// Is this view active and will recieve data /// public bool isActive { get; set; } /// /// Delete items on DocID before inserting new rows (default = true) /// public bool DeleteBeforeInsert { get; set; } /// /// Index in the background : better performance but reads might not have all the data /// public bool BackgroundIndexing { get; set; } /// /// Save documents to this view in the save process, like primary views /// public bool ConsistentSaveToThisView { get; set; } /// /// Apply to a Primary View and all the mappings of all views will be done in a transaction. /// You can use Rollback for failures. /// public bool TransactionMode { get; set; } /// /// When defining your own schema and you don't want dependancies to RaptorDB to propogate through your code /// define your full text columns here /// public List FullTextColumns; /// /// When defining your own schems and you don't want dependancies to RaptorDB to propogate through your code /// define your case insensitive columns here /// public List CaseInsensitiveColumns; public Dictionary StringIndexLength; /// /// Columns that you don't want to index /// public List NoIndexingColumns; } public class View : ViewBase { public View() { isActive = true; DeleteBeforeInsert = true; BackgroundIndexing = true; FullTextColumns = new List(); CaseInsensitiveColumns = new List(); StringIndexLength = new Dictionary(); NoIndexingColumns = new List(); isPrimaryList = true; ConsistentSaveToThisView = true; } /// /// Inline delegate for the mapper function used for quick applications /// [XmlIgnore] public MapFunctionDelgate Mapper { get; set; } public Result Verify() { if (Name == null || Name == "") throw new Exception("Name must be given"); if (Schema == null) throw new Exception("Schema must be defined"); if (Schema.IsSubclassOf(typeof(RDBSchema)) == false) { var pi = Schema.GetProperty("docid"); if (pi == null || pi.PropertyType != typeof(Guid)) { var fi = Schema.GetField("docid"); if( fi == null || fi.FieldType != typeof(Guid)) throw new Exception("The schema must be derived from RaptorDB.RDBSchema or must contain a 'docid' Guid field or property"); } } if (Mapper == null) throw new Exception("A map function must be defined"); if (TransactionMode == true && isPrimaryList == false) throw new Exception("Transaction mode can only be enabled on Primary Views"); // FEATURE : add more verifications return new Result(true); } } } ================================================ FILE: RaptorDB.Common/ZipStorer.cs ================================================ // ZipStorer, by Jaime Olivares // Website: zipstorer.codeplex.com // Version: 2.35 (March 14, 2010) using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Text; namespace RaptorDB.Common { public static class ZIP { #region [ usage sample ] static void Main(string[] args) { if (args.Length == 0) { printhelp(); return; } var t = args[0].ToLower(); var r = false; var fn = ""; var dir = ""; if (args[1].StartsWith("/") || args[1].StartsWith("-")) { r = true; fn = args[2]; dir = args[3]; } else { fn = args[1]; dir = args[2]; } if (t == "z") ZIP.Compress(fn, dir, r, log); else if (t == "u") ZIP.Decompress(fn, dir, log); else printhelp(); } private static void log(string msg) { Console.WriteLine(msg); } private static void printhelp() { Console.WriteLine("usage :"); Console.WriteLine(" zip : z /r filename.zip path"); Console.WriteLine(" unzip : u filename.zip path"); } #endregion public static void Compress(string filename, string folder, bool recursive, Action log) { ZipStorer zip; if (File.Exists(filename) == false) // Creates a new zip file zip = ZipStorer.Create(filename, ""); else // Opens existing zip file zip = ZipStorer.Open(filename, FileAccess.Write); zip.EncodeUTF8 = true; // Stores all the files into the zip file var dir = new DirectoryInfo(folder); var prefix = dir.FullName; if (prefix.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) prefix += Path.DirectorySeparatorChar; docompressdirectory(zip, dir.FullName, prefix, recursive, log); // Updates and closes the zip file zip.Close(); } public static void Decompress(string filename, string outputfolder, Action log) { ZipStorer zip; if (File.Exists(filename)) // Opens existing zip file zip = ZipStorer.Open(filename, FileAccess.Read); else return; // Read all directory contents List dir = zip.ReadCentralDir(); // Extract all files in target directory string path; bool result; foreach (ZipStorer.ZipFileEntry entry in dir) { path = Path.Combine(outputfolder, entry.FilenameInZip); result = zip.ExtractFile(entry, path); log?.Invoke(path); } zip.Close(); } private static void docompressdirectory(ZipStorer zip, string dir, string prefix, bool recursive, Action log) { if (recursive) foreach (var d in Directory.GetDirectories(dir)) docompressdirectory(zip, d, prefix, recursive, log); foreach (string path in Directory.GetFiles(dir)) { var fn = path.Replace(prefix, ""); zip.AddFile(ZipStorer.Compression.Deflate, path, fn, ""); log?.Invoke(fn); } } } /// /// Unique class for compression/decompression file. Represents a Zip file. /// public class ZipStorer : IDisposable { /// /// Compression method enumeration /// public enum Compression : ushort { /// Uncompressed storage Store = 0, /// Deflate compression method Deflate = 8 } /// /// Represents an entry in Zip file directory /// public struct ZipFileEntry { /// Compression method public Compression Method; /// Full path and filename as stored in Zip public string FilenameInZip; /// Original file size public uint FileSize; /// Compressed file size public uint CompressedSize; /// Offset of header information inside Zip storage public uint HeaderOffset; /// Offset of file inside Zip storage public uint FileOffset; /// Size of header information public uint HeaderSize; /// 32-bit checksum of entire file public uint Crc32; /// Last modification time of file public DateTime ModifyTime; /// User comment for file public string Comment; /// True if UTF8 encoding for filename and comments, false if default (CP 437) public bool EncodeUTF8; /// Overriden method /// Filename in Zip public override string ToString() { return this.FilenameInZip; } } #region Public fields /// True if UTF8 encoding for filename and comments, false if default (CP 437) public bool EncodeUTF8 = true; /// Force deflate algotithm even if it inflates the stored file. Off by default. public bool ForceDeflating = false; #endregion #region Private fields // List of files to store private List Files = new List(); // Filename of storage file private string FileName; // Stream object of storage file private Stream ZipFileStream; // General comment private string Comment = ""; // Central dir image private byte[] CentralDirImage = null; // Existing files in zip private ushort ExistingFiles = 0; // File access for Open method private FileAccess Access; // Static CRC32 Table private static UInt32[] CrcTable = null; // Default filename encoder private static Encoding DefaultEncoding = Encoding.GetEncoding(437); #endregion #region Public methods // Static constructor. Just invoked once in order to create the CRC32 lookup table. static ZipStorer() { // Generate CRC32 table CrcTable = new UInt32[256]; for (int i = 0; i < CrcTable.Length; i++) { UInt32 c = (UInt32)i; for (int j = 0; j < 8; j++) { if ((c & 1) != 0) c = 3988292384 ^ (c >> 1); else c >>= 1; } CrcTable[i] = c; } } /// /// Method to create a new storage file /// /// Full path of Zip file to create /// General comment for Zip file /// A valid ZipStorer object public static ZipStorer Create(string _filename, string _comment) { Stream stream = new FileStream(_filename, FileMode.Create, FileAccess.ReadWrite); ZipStorer zip = Create(stream, _comment); zip.Comment = _comment; zip.FileName = _filename; return zip; } /// /// Method to create a new zip storage in a stream /// /// /// /// A valid ZipStorer object public static ZipStorer Create(Stream _stream, string _comment) { ZipStorer zip = new ZipStorer(); zip.Comment = _comment; zip.ZipFileStream = _stream; zip.Access = FileAccess.Write; return zip; } /// /// Method to open an existing storage file /// /// Full path of Zip file to open /// File access mode as used in FileStream constructor /// A valid ZipStorer object public static ZipStorer Open(string _filename, FileAccess _access) { Stream stream = (Stream)new FileStream(_filename, FileMode.Open, _access == FileAccess.Read ? FileAccess.Read : FileAccess.ReadWrite); ZipStorer zip = Open(stream, _access); zip.FileName = _filename; return zip; } /// /// Method to open an existing storage from stream /// /// Already opened stream with zip contents /// File access mode for stream operations /// A valid ZipStorer object public static ZipStorer Open(Stream _stream, FileAccess _access) { if (!_stream.CanSeek && _access != FileAccess.Read) throw new InvalidOperationException("Stream cannot seek"); ZipStorer zip = new ZipStorer(); //zip.FileName = _filename; zip.ZipFileStream = _stream; zip.Access = _access; if (zip.ReadFileInfo()) return zip; throw new System.IO.InvalidDataException(); } /// /// Add full contents of a file into the Zip storage /// /// Compression method /// Full path of file to add to Zip storage /// Filename and path as desired in Zip directory /// Comment for stored file public void AddFile(Compression _method, string _pathname, string _filenameInZip, string _comment) { if (Access == FileAccess.Read) throw new InvalidOperationException("Writing is not alowed"); FileStream stream = new FileStream(_pathname, FileMode.Open, FileAccess.Read); AddStream(_method, _filenameInZip, stream, File.GetLastWriteTime(_pathname), _comment); stream.Close(); } public void AddFile(Compression _method, string _pathname, string _filenameInZip, string _comment, DateTime lastwrite) { if (Access == FileAccess.Read) throw new InvalidOperationException("Writing is not alowed"); FileStream stream = new FileStream(_pathname, FileMode.Open, FileAccess.Read); AddStream(_method, _filenameInZip, stream, lastwrite, _comment); stream.Close(); } /// /// Add full contents of a stream into the Zip storage /// /// Compression method /// Filename and path as desired in Zip directory /// Stream object containing the data to store in Zip /// Modification time of the data to store /// Comment for stored file public void AddStream(Compression _method, string _filenameInZip, Stream _source, DateTime _modTime, string _comment) { if (Access == FileAccess.Read) throw new InvalidOperationException("Writing is not alowed"); long offset; if (this.Files.Count == 0) offset = 0; else { ZipFileEntry last = this.Files[this.Files.Count - 1]; offset = last.HeaderOffset + last.HeaderSize; } // Prepare the fileinfo ZipFileEntry zfe = new ZipFileEntry(); zfe.Method = _method; zfe.EncodeUTF8 = this.EncodeUTF8; zfe.FilenameInZip = NormalizedFilename(_filenameInZip); zfe.Comment = (_comment == null ? "" : _comment); // Even though we write the header now, it will have to be rewritten, since we don't know compressed size or crc. zfe.Crc32 = 0; // to be updated later zfe.HeaderOffset = (uint)this.ZipFileStream.Position; // offset within file of the start of this local record zfe.ModifyTime = _modTime; // Write local header WriteLocalHeader(ref zfe); zfe.FileOffset = (uint)this.ZipFileStream.Position; // Write file to zip (store) Store(ref zfe, _source); _source.Close(); this.UpdateCrcAndSizes(ref zfe); Files.Add(zfe); } /// /// Updates central directory (if pertinent) and close the Zip storage /// /// This is a required step, unless automatic dispose is used public void Close() { if (this.Access != FileAccess.Read) { uint centralOffset = (uint)this.ZipFileStream.Position; uint centralSize = 0; if (this.CentralDirImage != null) this.ZipFileStream.Write(CentralDirImage, 0, CentralDirImage.Length); for (int i = 0; i < Files.Count; i++) { long pos = this.ZipFileStream.Position; this.WriteCentralDirRecord(Files[i]); centralSize += (uint)(this.ZipFileStream.Position - pos); } if (this.CentralDirImage != null) this.WriteEndRecord(centralSize + (uint)CentralDirImage.Length, centralOffset); else this.WriteEndRecord(centralSize, centralOffset); } if (this.ZipFileStream != null) { this.ZipFileStream.Flush(); this.ZipFileStream.Dispose(); this.ZipFileStream = null; } } /// /// Read all the file records in the central directory /// /// List of all entries in directory public List ReadCentralDir() { if (this.CentralDirImage == null) throw new InvalidOperationException("Central directory currently does not exist"); List result = new List(); for (int pointer = 0; pointer < this.CentralDirImage.Length;) { uint signature = BitConverter.ToUInt32(CentralDirImage, pointer); if (signature != 0x02014b50) break; bool encodeUTF8 = (BitConverter.ToUInt16(CentralDirImage, pointer + 8) & 0x0800) != 0; ushort method = BitConverter.ToUInt16(CentralDirImage, pointer + 10); uint modifyTime = BitConverter.ToUInt32(CentralDirImage, pointer + 12); uint crc32 = BitConverter.ToUInt32(CentralDirImage, pointer + 16); uint comprSize = BitConverter.ToUInt32(CentralDirImage, pointer + 20); uint fileSize = BitConverter.ToUInt32(CentralDirImage, pointer + 24); ushort filenameSize = BitConverter.ToUInt16(CentralDirImage, pointer + 28); ushort extraSize = BitConverter.ToUInt16(CentralDirImage, pointer + 30); ushort commentSize = BitConverter.ToUInt16(CentralDirImage, pointer + 32); uint headerOffset = BitConverter.ToUInt32(CentralDirImage, pointer + 42); uint headerSize = (uint)(46 + filenameSize + extraSize + commentSize); Encoding encoder = encodeUTF8 ? Encoding.UTF8 : DefaultEncoding; ZipFileEntry zfe = new ZipFileEntry(); zfe.Method = (Compression)method; zfe.FilenameInZip = encoder.GetString(CentralDirImage, pointer + 46, filenameSize); zfe.FileOffset = GetFileOffset(headerOffset); zfe.FileSize = fileSize; zfe.CompressedSize = comprSize; zfe.HeaderOffset = headerOffset; zfe.HeaderSize = headerSize; zfe.Crc32 = crc32; zfe.ModifyTime = DosTimeToDateTime(modifyTime); if (commentSize > 0) zfe.Comment = encoder.GetString(CentralDirImage, pointer + 46 + filenameSize + extraSize, commentSize); result.Add(zfe); pointer += (46 + filenameSize + extraSize + commentSize); } return result; } /// /// Copy the contents of a stored file into a physical file /// /// Entry information of file to extract /// Name of file to store uncompressed data /// True if success, false if not. /// Unique compression methods are Store and Deflate public bool ExtractFile(ZipFileEntry _zfe, string _filename) { // Make sure the parent directory exist string path = System.IO.Path.GetDirectoryName(_filename); if (!Directory.Exists(path)) Directory.CreateDirectory(path); // Check it is directory. If so, do nothing if (Directory.Exists(_filename)) return true; Stream output = new FileStream(_filename, FileMode.Create, FileAccess.Write); bool result = ExtractFile(_zfe, output); if (result) output.Close(); File.SetCreationTime(_filename, _zfe.ModifyTime); File.SetLastWriteTime(_filename, _zfe.ModifyTime); return result; } public bool ExtractFile2(ZipFileEntry _zfe, string _filename) { // Make sure the parent directory exist //string path = System.IO.Path.GetDirectoryName(_filename); //if (!Directory.Exists(path)) // Directory.CreateDirectory(path); //// Check it is directory. If so, do nothing //if (Directory.Exists(_filename)) // return true; Stream output = new FileStream(_filename, FileMode.Create, FileAccess.Write); bool result = ExtractFile(_zfe, output); if (result) output.Close(); File.SetCreationTime(_filename, _zfe.ModifyTime); File.SetLastWriteTime(_filename, _zfe.ModifyTime); return result; } /// /// Copy the contents of a stored file into an opened stream /// /// Entry information of file to extract /// Stream to store the uncompressed data /// True if success, false if not. /// Unique compression methods are Store and Deflate public bool ExtractFile(ZipFileEntry _zfe, Stream _stream) { if (!_stream.CanWrite) throw new InvalidOperationException("Stream cannot be written"); // check signature byte[] signature = new byte[4]; this.ZipFileStream.Seek(_zfe.HeaderOffset, SeekOrigin.Begin); this.ZipFileStream.Read(signature, 0, 4); if (BitConverter.ToUInt32(signature, 0) != 0x04034b50) return false; // Select input stream for inflating or just reading Stream inStream; if (_zfe.Method == Compression.Store) inStream = this.ZipFileStream; else if (_zfe.Method == Compression.Deflate) inStream = new DeflateStream(this.ZipFileStream, CompressionMode.Decompress, true); else return false; // Buffered copy byte[] buffer = new byte[16384]; this.ZipFileStream.Seek(_zfe.FileOffset, SeekOrigin.Begin); uint bytesPending = _zfe.FileSize; while (bytesPending > 0) { int bytesRead = inStream.Read(buffer, 0, (int)Math.Min(bytesPending, buffer.Length)); _stream.Write(buffer, 0, bytesRead); bytesPending -= (uint)bytesRead; } _stream.Flush(); if (_zfe.Method == Compression.Deflate) inStream.Dispose(); return true; } /// /// Removes one of many files in storage. It creates a new Zip file. /// /// Reference to the current Zip object /// List of Entries to remove from storage /// True if success, false if not /// This method only works for storage of type FileStream public static bool RemoveEntries(ref ZipStorer _zip, List _zfes) { if (!(_zip.ZipFileStream is FileStream)) throw new InvalidOperationException("RemoveEntries is allowed just over streams of type FileStream"); //Get full list of entries List fullList = _zip.ReadCentralDir(); //In order to delete we need to create a copy of the zip file excluding the selected items string tempZipName = Path.GetTempFileName(); string tempEntryName = Path.GetTempFileName(); try { ZipStorer tempZip = ZipStorer.Create(tempZipName, string.Empty); foreach (ZipFileEntry zfe in fullList) { if (!_zfes.Contains(zfe)) { if (_zip.ExtractFile(zfe, tempEntryName)) { tempZip.AddFile(zfe.Method, tempEntryName, zfe.FilenameInZip, zfe.Comment); } } } _zip.Close(); tempZip.Close(); File.Delete(_zip.FileName); File.Move(tempZipName, _zip.FileName); _zip = ZipStorer.Open(_zip.FileName, _zip.Access); } catch { return false; } finally { if (File.Exists(tempZipName)) File.Delete(tempZipName); if (File.Exists(tempEntryName)) File.Delete(tempEntryName); } return true; } #endregion #region Private methods // Calculate the file offset by reading the corresponding local header private uint GetFileOffset(uint _headerOffset) { byte[] buffer = new byte[2]; this.ZipFileStream.Seek(_headerOffset + 26, SeekOrigin.Begin); this.ZipFileStream.Read(buffer, 0, 2); ushort filenameSize = BitConverter.ToUInt16(buffer, 0); this.ZipFileStream.Read(buffer, 0, 2); ushort extraSize = BitConverter.ToUInt16(buffer, 0); return (uint)(30 + filenameSize + extraSize + _headerOffset); } /* Local file header: local file header signature 4 bytes (0x04034b50) version needed to extract 2 bytes general purpose bit flag 2 bytes compression method 2 bytes last mod file time 2 bytes last mod file date 2 bytes crc-32 4 bytes compressed size 4 bytes uncompressed size 4 bytes filename length 2 bytes extra field length 2 bytes filename (variable size) extra field (variable size) */ private void WriteLocalHeader(ref ZipFileEntry _zfe) { long pos = this.ZipFileStream.Position; Encoding encoder = _zfe.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding; byte[] encodedFilename = encoder.GetBytes(_zfe.FilenameInZip); this.ZipFileStream.Write(new byte[] { 80, 75, 3, 4, 20, 0 }, 0, 6); // No extra header this.ZipFileStream.Write(BitConverter.GetBytes((ushort)(_zfe.EncodeUTF8 ? 0x0800 : 0)), 0, 2); // filename and comment encoding this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method this.ZipFileStream.Write(BitConverter.GetBytes(DateTimeToDosTime(_zfe.ModifyTime)), 0, 4); // zipping date and time this.ZipFileStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 12); // unused CRC, un/compressed size, updated later this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedFilename.Length), 0, 2); // filename length this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // extra length this.ZipFileStream.Write(encodedFilename, 0, encodedFilename.Length); _zfe.HeaderSize = (uint)(this.ZipFileStream.Position - pos); } /* Central directory's File header: central file header signature 4 bytes (0x02014b50) version made by 2 bytes version needed to extract 2 bytes general purpose bit flag 2 bytes compression method 2 bytes last mod file time 2 bytes last mod file date 2 bytes crc-32 4 bytes compressed size 4 bytes uncompressed size 4 bytes filename length 2 bytes extra field length 2 bytes file comment length 2 bytes disk number start 2 bytes internal file attributes 2 bytes external file attributes 4 bytes relative offset of local header 4 bytes filename (variable size) extra field (variable size) file comment (variable size) */ private void WriteCentralDirRecord(ZipFileEntry _zfe) { Encoding encoder = _zfe.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding; byte[] encodedFilename = encoder.GetBytes(_zfe.FilenameInZip); byte[] encodedComment = encoder.GetBytes(_zfe.Comment); this.ZipFileStream.Write(new byte[] { 80, 75, 1, 2, 23, 0xB, 20, 0 }, 0, 8); this.ZipFileStream.Write(BitConverter.GetBytes((ushort)(_zfe.EncodeUTF8 ? 0x0800 : 0)), 0, 2); // filename and comment encoding this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method this.ZipFileStream.Write(BitConverter.GetBytes(DateTimeToDosTime(_zfe.ModifyTime)), 0, 4); // zipping date and time this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.Crc32), 0, 4); // file CRC this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.CompressedSize), 0, 4); // compressed file size this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.FileSize), 0, 4); // uncompressed file size this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedFilename.Length), 0, 2); // Filename in zip this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // extra length this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedComment.Length), 0, 2); this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // disk=0 this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // file type: binary this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // Internal file attributes this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0x8100), 0, 2); // External file attributes (normal/readable) this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.HeaderOffset), 0, 4); // Offset of header this.ZipFileStream.Write(encodedFilename, 0, encodedFilename.Length); this.ZipFileStream.Write(encodedComment, 0, encodedComment.Length); } /* End of central dir record: end of central dir signature 4 bytes (0x06054b50) number of this disk 2 bytes number of the disk with the start of the central directory 2 bytes total number of entries in the central dir on this disk 2 bytes total number of entries in the central dir 2 bytes size of the central directory 4 bytes offset of start of central directory with respect to the starting disk number 4 bytes zipfile comment length 2 bytes zipfile comment (variable size) */ private void WriteEndRecord(uint _size, uint _offset) { Encoding encoder = this.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding; byte[] encodedComment = encoder.GetBytes(this.Comment); this.ZipFileStream.Write(new byte[] { 80, 75, 5, 6, 0, 0, 0, 0 }, 0, 8); this.ZipFileStream.Write(BitConverter.GetBytes((ushort)Files.Count + ExistingFiles), 0, 2); this.ZipFileStream.Write(BitConverter.GetBytes((ushort)Files.Count + ExistingFiles), 0, 2); this.ZipFileStream.Write(BitConverter.GetBytes(_size), 0, 4); this.ZipFileStream.Write(BitConverter.GetBytes(_offset), 0, 4); this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedComment.Length), 0, 2); this.ZipFileStream.Write(encodedComment, 0, encodedComment.Length); } // Copies all source file into storage file private void Store(ref ZipFileEntry _zfe, Stream _source) { byte[] buffer = new byte[16384]; int bytesRead; uint totalRead = 0; Stream outStream; long posStart = this.ZipFileStream.Position; long sourceStart = _source.Position; if (_zfe.Method == Compression.Store) outStream = this.ZipFileStream; else outStream = new DeflateStream(this.ZipFileStream, CompressionMode.Compress, true); _zfe.Crc32 = 0 ^ 0xffffffff; do { bytesRead = _source.Read(buffer, 0, buffer.Length); totalRead += (uint)bytesRead; if (bytesRead > 0) { outStream.Write(buffer, 0, bytesRead); for (uint i = 0; i < bytesRead; i++) { _zfe.Crc32 = ZipStorer.CrcTable[(_zfe.Crc32 ^ buffer[i]) & 0xFF] ^ (_zfe.Crc32 >> 8); } } } while (bytesRead == buffer.Length); outStream.Flush(); if (_zfe.Method == Compression.Deflate) outStream.Dispose(); _zfe.Crc32 ^= 0xffffffff; _zfe.FileSize = totalRead; _zfe.CompressedSize = (uint)(this.ZipFileStream.Position - posStart); // Verify for real compression if (_zfe.Method == Compression.Deflate && !this.ForceDeflating && _source.CanSeek && _zfe.CompressedSize > _zfe.FileSize) { // Start operation again with Store algorithm _zfe.Method = Compression.Store; this.ZipFileStream.Position = posStart; this.ZipFileStream.SetLength(posStart); _source.Position = sourceStart; this.Store(ref _zfe, _source); } } /* DOS Date and time: MS-DOS date. The date is a packed value with the following format. Bits Description 0-4 Day of the month (1�31) 5-8 Month (1 = January, 2 = February, and so on) 9-15 Year offset from 1980 (add 1980 to get actual year) MS-DOS time. The time is a packed value with the following format. Bits Description 0-4 Second divided by 2 5-10 Minute (0�59) 11-15 Hour (0�23 on a 24-hour clock) */ private uint DateTimeToDosTime(DateTime _dt) { return (uint)( (_dt.Second / 2) | (_dt.Minute << 5) | (_dt.Hour << 11) | (_dt.Day << 16) | (_dt.Month << 21) | ((_dt.Year - 1980) << 25)); } private DateTime DosTimeToDateTime(uint _dt) { return new DateTime( (int)(_dt >> 25) + 1980, (int)(_dt >> 21) & 15, (int)(_dt >> 16) & 31, (int)(_dt >> 11) & 31, (int)(_dt >> 5) & 63, (int)(_dt & 31) * 2); } /* CRC32 algorithm The 'magic number' for the CRC is 0xdebb20e3. The proper CRC pre and post conditioning is used, meaning that the CRC register is pre-conditioned with all ones (a starting value of 0xffffffff) and the value is post-conditioned by taking the one's complement of the CRC residual. If bit 3 of the general purpose flag is set, this field is set to zero in the local header and the correct value is put in the data descriptor and in the central directory. */ private void UpdateCrcAndSizes(ref ZipFileEntry _zfe) { long lastPos = this.ZipFileStream.Position; // remember position this.ZipFileStream.Position = _zfe.HeaderOffset + 8; this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method this.ZipFileStream.Position = _zfe.HeaderOffset + 14; this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.Crc32), 0, 4); // Update CRC this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.CompressedSize), 0, 4); // Compressed size this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.FileSize), 0, 4); // Uncompressed size this.ZipFileStream.Position = lastPos; // restore position } // Replaces backslashes with slashes to store in zip header private string NormalizedFilename(string _filename) { string filename = _filename.Replace('\\', '/'); int pos = filename.IndexOf(':'); if (pos >= 0) filename = filename.Remove(0, pos + 1); return filename.Trim('/'); } // Reads the end-of-central-directory record private bool ReadFileInfo() { if (this.ZipFileStream.Length < 22) return false; try { this.ZipFileStream.Seek(-17, SeekOrigin.End); BinaryReader br = new BinaryReader(this.ZipFileStream); do { this.ZipFileStream.Seek(-5, SeekOrigin.Current); UInt32 sig = br.ReadUInt32(); if (sig == 0x06054b50) { this.ZipFileStream.Seek(6, SeekOrigin.Current); UInt16 entries = br.ReadUInt16(); Int32 centralSize = br.ReadInt32(); UInt32 centralDirOffset = br.ReadUInt32(); UInt16 commentSize = br.ReadUInt16(); // check if comment field is the very last data in file if (this.ZipFileStream.Position + commentSize != this.ZipFileStream.Length) return false; // Copy entire central directory to a memory buffer this.ExistingFiles = entries; this.CentralDirImage = new byte[centralSize]; this.ZipFileStream.Seek(centralDirOffset, SeekOrigin.Begin); this.ZipFileStream.Read(this.CentralDirImage, 0, centralSize); // Leave the pointer at the begining of central dir, to append new files this.ZipFileStream.Seek(centralDirOffset, SeekOrigin.Begin); return true; } } while (this.ZipFileStream.Position > 0); } catch { } return false; } #endregion #region IDisposable Members /// /// Closes the Zip file stream /// public void Dispose() { this.Close(); } #endregion } } ================================================ FILE: RaptorDB.Common/fastBinaryJSON/BJSON.cs ================================================ using System; using System.Collections; using System.Collections.Generic; #if !SILVERLIGHT using System.Data; #endif using System.IO; using System.Collections.Specialized; using fastJSON; namespace fastBinaryJSON { public sealed class TOKENS { public const byte DOC_START = 1; public const byte DOC_END = 2; public const byte ARRAY_START = 3; public const byte ARRAY_END = 4; public const byte COLON = 5; public const byte COMMA = 6; public const byte NAME = 7; public const byte STRING = 8; public const byte BYTE = 9; public const byte INT = 10; public const byte UINT = 11; public const byte LONG = 12; public const byte ULONG = 13; public const byte SHORT = 14; public const byte USHORT = 15; public const byte DATETIME = 16; public const byte GUID = 17; public const byte DOUBLE = 18; public const byte FLOAT = 19; public const byte DECIMAL = 20; public const byte CHAR = 21; public const byte BYTEARRAY = 22; public const byte NULL = 23; public const byte TRUE = 24; public const byte FALSE = 25; public const byte UNICODE_STRING = 26; public const byte DATETIMEOFFSET = 27; public const byte ARRAY_TYPED = 28; public const byte TYPES_POINTER = 29; public const byte TIMESPAN = 30; public const byte ARRAY_TYPED_LONG = 31; public const byte NAME_UNI = 32; } public class typedarray { public string typename; public int count; public List data = new List(); } public sealed class BJSONParameters { /// /// Optimize the schema for Datasets (default = True) /// public bool UseOptimizedDatasetSchema = true; /// /// Serialize readonly properties (default = False) /// public bool ShowReadOnlyProperties = false; /// /// Use global types $types for more compact size when using a lot of classes (default = True) /// public bool UsingGlobalTypes = true; /// /// Use Unicode strings = T (faster), Use UTF8 strings = F (smaller) (default = True) /// public bool UseUnicodeStrings = true; /// /// Serialize Null values to the output (default = False) /// public bool SerializeNulls = false; /// /// Enable fastBinaryJSON extensions $types, $type, $map (default = True) /// public bool UseExtensions = true; /// /// Anonymous types have read only properties /// public bool EnableAnonymousTypes = false; /// /// Use the UTC date format (default = False) /// public bool UseUTCDateTime = false; /// /// Ignore attributes to check for (default : XmlIgnoreAttribute, NonSerialized) /// public List IgnoreAttributes = new List { typeof(System.Xml.Serialization.XmlIgnoreAttribute), typeof(NonSerializedAttribute) }; /// /// If you have parametric and no default constructor for you classes (default = False) /// /// IMPORTANT NOTE : If True then all initial values within the class will be ignored and will be not set /// public bool ParametricConstructorOverride = false; /// /// Maximum depth the serializer will go to to avoid loops (default = 20 levels) /// public short SerializerMaxDepth = 20; /// /// Use typed arrays t[] into object = t[] not object[] (default = true) /// public bool UseTypedArrays = true; /// /// Backward compatible Typed array type name as UTF8 (default = false -> fast v1.5 unicode) /// public bool v1_4TypedArray = false; public void FixValues() { if (UseExtensions == false) // disable conflicting params UsingGlobalTypes = false; if (EnableAnonymousTypes) ShowReadOnlyProperties = true; } internal BJSONParameters MakeCopy() { return new BJSONParameters { UseOptimizedDatasetSchema = UseOptimizedDatasetSchema, ShowReadOnlyProperties = ShowReadOnlyProperties, EnableAnonymousTypes = EnableAnonymousTypes, UsingGlobalTypes = UsingGlobalTypes, IgnoreAttributes = new List(IgnoreAttributes), UseUnicodeStrings = UseUnicodeStrings, SerializeNulls = SerializeNulls, ParametricConstructorOverride = ParametricConstructorOverride, SerializerMaxDepth = SerializerMaxDepth, UseTypedArrays = UseTypedArrays, UseExtensions = UseExtensions, UseUTCDateTime = UseUTCDateTime, v1_4TypedArray = v1_4TypedArray//, //OptimizeSize = OptimizeSize }; } } public static class BJSON { /// /// Globally set-able parameters for controlling the serializer /// public static BJSONParameters Parameters = new BJSONParameters(); /// /// Parse a json and generate a Dictionary<string,object> or List<object> structure /// /// /// public static object Parse(byte[] json) { return new BJsonParser(json, Parameters.UseUTCDateTime, Parameters.v1_4TypedArray).Decode(); } #if NET4 /// /// Create a .net4 dynamic object from the binary json byte array /// /// /// public static dynamic ToDynamic(byte[] json) { return new DynamicJson(json); } #endif /// /// Register custom type handlers for your own types not natively handled by fastBinaryJSON /// /// /// /// public static void RegisterCustomType(Type type, Reflection.Serialize serializer, Reflection.Deserialize deserializer) { Reflection.Instance.RegisterCustomType(type, serializer, deserializer); } /// /// Create a binary json representation for an object /// /// /// public static byte[] ToBJSON(object obj) { return ToBJSON(obj, Parameters); } /// /// Create a binary json representation for an object with parameter override on this call /// /// /// /// public static byte[] ToBJSON(object obj, BJSONParameters param) { param.FixValues(); param = param.MakeCopy(); Type t = null; if (obj == null) return new byte[] { TOKENS.NULL }; if (obj.GetType().IsGenericType) t = Reflection.Instance.GetGenericTypeDefinition(obj.GetType());// obj.GetType().GetGenericTypeDefinition(); if (t == typeof(Dictionary<,>) || t == typeof(List<>)) param.UsingGlobalTypes = false; // FEATURE : enable extensions when you can deserialize anon types if (param.EnableAnonymousTypes) { param.UseExtensions = false; param.UsingGlobalTypes = false; } return new BJSONSerializer(param).ConvertToBJSON(obj); } /// /// Fill a given object with the binary json represenation /// /// /// /// public static object FillObject(object input, byte[] json) { return new deserializer(Parameters).FillObject(input, json); } /// /// Create a generic object from the json /// /// /// /// public static T ToObject(byte[] json) { return new deserializer(Parameters).ToObject(json); } /// /// Create a generic object from the json with parameter override on this call /// /// /// /// /// public static T ToObject(byte[] json, BJSONParameters param) { return new deserializer(param).ToObject(json); } /// /// Create an object from the json /// /// /// public static object ToObject(byte[] json) { return new deserializer(Parameters).ToObject(json, null); } /// /// Create an object from the json with parameter override on this call /// /// /// /// public static object ToObject(byte[] json, BJSONParameters param) { param.FixValues(); param = param.MakeCopy(); return new deserializer(param).ToObject(json, null); } /// /// Create a typed object from the json /// /// /// /// public static object ToObject(byte[] json, Type type) { return new deserializer(Parameters).ToObject(json, type); } /// /// Clear the internal reflection cache so you can start from new (you will loose performance) /// public static void ClearReflectionCache() { Reflection.Instance.ClearReflectionCache(); } /// /// Deep copy an object i.e. clone to a new object /// /// /// public static object DeepCopy(object obj) { return new deserializer(Parameters).ToObject(ToBJSON(obj)); } } internal class deserializer { public deserializer(BJSONParameters param) { _params = param; _params = param.MakeCopy(); } private BJSONParameters _params; private Dictionary _circobj = new Dictionary(); private Dictionary _cirrev = new Dictionary(); public T ToObject(byte[] json) { return (T)ToObject(json, typeof(T)); } public object ToObject(byte[] json) { return ToObject(json, null); } public object ToObject(byte[] json, Type type) { //_params.FixValues(); Type t = null; if (type != null && type.IsGenericType) t = Reflection.Instance.GetGenericTypeDefinition(type);// type.GetGenericTypeDefinition(); _globalTypes = _params.UsingGlobalTypes; if (t == typeof(Dictionary<,>) || t == typeof(List<>)) _globalTypes = false; var o = new BJsonParser(json, _params.UseUTCDateTime, _params.v1_4TypedArray).Decode(); if (type?.IsEnum == true) return CreateEnum(type, o); #if !SILVERLIGHT if (type != null && type == typeof(DataSet)) return CreateDataset(o as Dictionary, null); if (type != null && type == typeof(DataTable)) return CreateDataTable(o as Dictionary, null); #endif if (o is typedarray) { return ParseTypedArray(new Dictionary(), o); } if (o is IDictionary) { if (type != null && t == typeof(Dictionary<,>)) // deserialize a dictionary return RootDictionary(o, type); else // deserialize an object return ParseDictionary(o as Dictionary, null, type, null); } if (o is List) { if (type != null && t == typeof(Dictionary<,>)) // kv format return RootDictionary(o, type); if (type != null && t == typeof(List<>)) // deserialize to generic list return RootList(o, type); if (type == typeof(Hashtable)) return RootHashTable((List)o); else if (type == null) { List l = (List)o; if (l.Count > 0 && l[0].GetType() == typeof(Dictionary)) { Dictionary globals = new Dictionary(); List op = new List(); // try to get $types foreach (var i in l) op.Add(ParseDictionary((Dictionary)i, globals, null, null)); return op; } return l.ToArray(); } } else if (type != null && o.GetType() != type) return ChangeType(o, type); return o; } private object ChangeType(object o, Type type) { if (Reflection.Instance.IsTypeRegistered(type)) return Reflection.Instance.CreateCustom((string)o, type); else return o; } public object FillObject(object input, byte[] json) { _params.FixValues(); Dictionary ht = new BJsonParser(json, _params.UseUTCDateTime, _params.v1_4TypedArray).Decode() as Dictionary; if (ht == null) return null; return ParseDictionary(ht, null, input.GetType(), input); } private object RootHashTable(List o) { Hashtable h = new Hashtable(); foreach (Dictionary values in o) { object key = values["k"]; object val = values["v"]; if (key is Dictionary) key = ParseDictionary((Dictionary)key, null, typeof(object), null); if (val is Dictionary) val = ParseDictionary((Dictionary)val, null, typeof(object), null); h.Add(key, val); } return h; } private object RootList(object parse, Type type) { Type[] gtypes = Reflection.Instance.GetGenericArguments(type);// type.GetGenericArguments(); IList o = (IList)Reflection.Instance.FastCreateList(type, ((IList)parse).Count); Dictionary globals = new Dictionary(); foreach (var k in (IList)parse) { _globalTypes = false; object v = k; if (k is Dictionary) v = ParseDictionary(k as Dictionary, globals, gtypes[0], null); else v = k; o.Add(v); } return o; } private object RootDictionary(object parse, Type type) { Type[] gtypes = Reflection.Instance.GetGenericArguments(type); Type t1 = null; Type t2 = null; if (gtypes != null) { t1 = gtypes[0]; t2 = gtypes[1]; } var arraytype = t2.GetElementType(); if (parse is Dictionary) { IDictionary o = (IDictionary)Reflection.Instance.FastCreateInstance(type); foreach (var kv in (Dictionary)parse) { _globalTypes = false; object v; object k = kv.Key; if (t2.Name.StartsWith("Dictionary")) // deserialize a dictionary v = RootDictionary(kv.Value, t2); else if (kv.Value is Dictionary) v = ParseDictionary(kv.Value as Dictionary, null, t2, null); else if (t2 == typeof(byte[])) v = kv.Value; else if (gtypes != null && t2.IsArray) v = CreateArray((List)kv.Value, t2, arraytype, null); else if (kv.Value is IList) v = CreateGenericList((List)kv.Value, t2, t1, null); else v = kv.Value; o.Add(k, v); } return o; } if (parse is List) return CreateDictionary(parse as List, type, gtypes, null); return null; } private bool _globalTypes = false; private object ParseDictionary(Dictionary d, Dictionary globaltypes, Type type, object input) { object tn = ""; if (type == typeof(NameValueCollection)) return CreateNV(d); if (type == typeof(StringDictionary)) return CreateSD(d); if (d.TryGetValue("$i", out tn)) { object v = null; _cirrev.TryGetValue((int)tn, out v); return v; } if (d.TryGetValue("$types", out tn)) { _globalTypes = true; if (globaltypes == null) globaltypes = new Dictionary(); foreach (var kv in (Dictionary)tn) { globaltypes.Add((string)kv.Key, kv.Value); } } if (globaltypes != null) _globalTypes = true; bool found = d.TryGetValue("$type", out tn); #if !SILVERLIGHT if (found == false && type == typeof(System.Object)) { return d; // CreateDataset(d, globaltypes); } #endif if (found) { if (_globalTypes && globaltypes != null) { object tname = ""; if (globaltypes != null && globaltypes.TryGetValue((string)tn, out tname)) tn = tname; } type = Reflection.Instance.GetTypeFromCache((string)tn, true); } if (type == null) throw new Exception("Cannot determine type"); string typename = type.FullName; object o = input; if (o == null) { if (_params.ParametricConstructorOverride) o = System.Runtime.Serialization.FormatterServices.GetUninitializedObject(type); else o = Reflection.Instance.FastCreateInstance(type); } int circount = 0; if (_circobj.TryGetValue(o, out circount) == false) { circount = _circobj.Count + 1; _circobj.Add(o, circount); _cirrev.Add(circount, o); } Dictionary props = Reflection.Instance.Getproperties(type, typename, _params.ShowReadOnlyProperties);//, Reflection.Instance.IsTypeRegistered(type)); foreach (var kv in d) { var n = kv.Key; var v = kv.Value; string name = n.ToLowerInvariant(); myPropInfo pi; if (props.TryGetValue(name, out pi) == false) continue; if (pi.CanWrite) { //object v = d[n]; if (v != null) { object oset = v; if (v is typedarray) { oset = ParseTypedArray(globaltypes, v); } else { switch (pi.Type) { #if !SILVERLIGHT case myPropInfoType.DataSet: oset = CreateDataset((Dictionary)v, globaltypes); break; case myPropInfoType.DataTable: oset = CreateDataTable((Dictionary)v, globaltypes); break; #endif case myPropInfoType.Custom: oset = Reflection.Instance.CreateCustom((string)v, pi.pt); break; case myPropInfoType.Enum: oset = CreateEnum(pi.pt, v); break; case myPropInfoType.StringKeyDictionary: oset = CreateStringKeyDictionary((Dictionary)v, pi.pt, pi.GenericTypes, globaltypes); break; case myPropInfoType.Hashtable: case myPropInfoType.Dictionary: oset = CreateDictionary((List)v, pi.pt, pi.GenericTypes, globaltypes); break; case myPropInfoType.NameValue: oset = CreateNV((Dictionary)v); break; case myPropInfoType.StringDictionary: oset = CreateSD((Dictionary)v); break; case myPropInfoType.Array: oset = CreateArray((List)v, pi.pt, pi.bt, globaltypes); break; default: { if (pi.IsGenericType && pi.IsValueType == false) oset = CreateGenericList((List)v, pi.pt, pi.bt, globaltypes); else if ((pi.IsClass || pi.IsStruct || pi.IsInterface) && v is Dictionary) { var oo = (Dictionary)v; if (oo.ContainsKey("$schema")) oset = CreateDataset(oo, globaltypes); else oset = ParseDictionary(oo, globaltypes, pi.pt, input); } else if (v is List) oset = CreateArray((List)v, pi.pt, typeof(object), globaltypes); break; } } } o = pi.setter(o, oset); } } } return o; } private object ParseTypedArray(Dictionary globaltypes, object v) { object oset; var ta = (typedarray)v; var t = Reflection.Instance.GetTypeFromCache(ta.typename, true); IList a = Array.CreateInstance(t, ta.count); int i = 0; foreach (var dd in ta.data) { object oo = null; if (dd == null) oo = null; else if (dd is typedarray) oo = ParseTypedArray(globaltypes, dd); else if (dd is Dictionary) oo = ParseDictionary((Dictionary)dd, globaltypes, t, null); else if (dd is List) oo = CreateArray((List)dd, t, t.GetElementType(), globaltypes); else oo = dd; a[i++] = oo; } oset = a; return oset; } private StringDictionary CreateSD(Dictionary d) { StringDictionary nv = new StringDictionary(); foreach (var o in d) nv.Add(o.Key, (string)o.Value); return nv; } private NameValueCollection CreateNV(Dictionary d) { NameValueCollection nv = new NameValueCollection(); foreach (var o in d) nv.Add(o.Key, (string)o.Value); return nv; } private object CreateEnum(Type pt, object v) { // FEATURE : optimize create enum #if !SILVERLIGHT return Enum.Parse(pt, v.ToString()); #else return Enum.Parse(pt, v, true); #endif } private object CreateArray(List data, Type pt, Type bt, Dictionary globalTypes) { if (bt == null) bt = typeof(object); Array col = Array.CreateInstance(bt, data.Count); var arraytype = bt.GetElementType(); // create an array of objects for (int i = 0; i < data.Count; i++)// each (object ob in data) { object ob = data[i]; if (ob == null) { continue; } if (ob is IDictionary) col.SetValue(ParseDictionary((Dictionary)ob, globalTypes, bt, null), i); else if (ob is ICollection) col.SetValue(CreateArray((List)ob, bt, arraytype, globalTypes), i); else col.SetValue(ob, i); } return col; } private object CreateGenericList(List data, Type pt, Type bt, Dictionary globalTypes) { if (pt != typeof(object)) { IList col = (IList)Reflection.Instance.FastCreateList(pt, data.Count); // create an array of objects foreach (object ob in data) { if (ob is IDictionary) col.Add(ParseDictionary((Dictionary)ob, globalTypes, bt, null)); else if (ob is List) { if (bt.IsGenericType) col.Add((List)ob); else col.Add(((List)ob).ToArray()); } else if(ob is typedarray) col.Add(((typedarray)ob).data.ToArray()); else col.Add(ob); } return col; } return data; } private object CreateStringKeyDictionary(Dictionary reader, Type pt, Type[] types, Dictionary globalTypes) { var col = (IDictionary)Reflection.Instance.FastCreateInstance(pt); Type arraytype = null; Type t2 = null; if (types != null) t2 = types[1]; Type generictype = null; var ga = Reflection.Instance.GetGenericArguments(t2); if (ga.Length > 0) generictype = ga[0]; arraytype = t2.GetElementType(); foreach (KeyValuePair values in reader) { var key = values.Key; object val = null; if (values.Value is Dictionary) val = ParseDictionary((Dictionary)values.Value, globalTypes, t2, null); else if (types != null && t2.IsArray) { if (values.Value is Array) val = values.Value; else val = CreateArray((List)values.Value, t2, arraytype, globalTypes); } else if (values.Value is IList) val = CreateGenericList((List)values.Value, t2, generictype, globalTypes); else val = values.Value; col.Add(key, val); } return col; } private object CreateDictionary(List reader, Type pt, Type[] types, Dictionary globalTypes) { IDictionary col = (IDictionary)Reflection.Instance.FastCreateInstance(pt); Type t1 = null; Type t2 = null; if (types != null) { t1 = types[0]; t2 = types[1]; } foreach (Dictionary values in reader) { object key = values["k"]; object val = values["v"]; if (key is Dictionary) key = ParseDictionary((Dictionary)key, globalTypes, t1, null); if (typeof(IDictionary).IsAssignableFrom(t2)) val = RootDictionary(val, t2); else if (val is Dictionary) val = ParseDictionary((Dictionary)val, globalTypes, t2, null); col.Add(key, val); } return col; } #if !SILVERLIGHT private DataSet CreateDataset(Dictionary reader, Dictionary globalTypes) { DataSet ds = new DataSet(); ds.EnforceConstraints = false; ds.BeginInit(); // read dataset schema here var schema = reader["$schema"]; if (schema is string) { TextReader tr = new StringReader((string)schema); ds.ReadXmlSchema(tr); } else { DatasetSchema ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null); ds.DataSetName = ms.Name; for (int i = 0; i < ms.Info.Count; i += 3) { if (ds.Tables.Contains(ms.Info[i]) == false) ds.Tables.Add(ms.Info[i]); ds.Tables[ms.Info[i]].Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2])); } } foreach (KeyValuePair pair in reader) { if (pair.Key == "$type" || pair.Key == "$schema") continue; List rows = (List)pair.Value; if (rows == null) continue; DataTable dt = ds.Tables[pair.Key]; ReadDataTable(rows, dt); } ds.EndInit(); return ds; } private void ReadDataTable(List rows, DataTable dt) { dt.BeginInit(); dt.BeginLoadData(); foreach (List row in rows) { object[] v = new object[row.Count]; row.CopyTo(v, 0); dt.Rows.Add(v); } dt.EndLoadData(); dt.EndInit(); } DataTable CreateDataTable(Dictionary reader, Dictionary globalTypes) { var dt = new DataTable(); // read dataset schema here var schema = reader["$schema"]; if (schema is string) { TextReader tr = new StringReader((string)schema); dt.ReadXmlSchema(tr); } else { var ms = (DatasetSchema)this.ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null); dt.TableName = ms.Info[0]; for (int i = 0; i < ms.Info.Count; i += 3) { dt.Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2])); } } foreach (var pair in reader) { if (pair.Key == "$type" || pair.Key == "$schema") continue; var rows = (List)pair.Value; if (rows == null) continue; if (!dt.TableName.Equals(pair.Key, StringComparison.InvariantCultureIgnoreCase)) continue; ReadDataTable(rows, dt); } return dt; } #endif } } ================================================ FILE: RaptorDB.Common/fastBinaryJSON/BJsonParser.cs ================================================ using fastJSON; using System; using System.Collections.Generic; namespace fastBinaryJSON { internal sealed class BJsonParser { readonly byte[] _json; int _index; bool _useUTC = true; bool _v1_4TA = false; internal BJsonParser(byte[] json, bool useUTC, bool v1_4TA) { this._json = json; _v1_4TA = v1_4TA; _useUTC = useUTC; } public object Decode() { bool b = false; return ParseValue(out b); } private Dictionary ParseObject() { Dictionary dic = new Dictionary(10); bool breakparse = false; while (!breakparse) { byte t = GetToken(); if (t == TOKENS.COMMA) continue; if (t == TOKENS.DOC_END) break; if (t == TOKENS.TYPES_POINTER) { // save curr index position int savedindex = _index; // set index = pointer _index = ParseInt(); t = GetToken(); // read $types breakparse = readkeyvalue(dic, ref t); // set index = saved + 4 _index = savedindex + 4; } else breakparse = readkeyvalue(dic, ref t); } return dic; } private bool readkeyvalue(Dictionary dic, ref byte t) { bool breakparse; string key = ""; //if (t != TOKENS.NAME) if (t == TOKENS.NAME) key = ParseName(); else if (t == TOKENS.NAME_UNI) key = ParseName2(); else throw new Exception("excpecting a name field"); t = GetToken(); if (t != TOKENS.COLON) throw new Exception("expecting a colon"); object val = ParseValue(out breakparse); if (breakparse == false) dic.Add(key, val); return breakparse; } private string ParseName2() // unicode byte len string -> <128 len chars { byte c = _json[_index++]; string s = Reflection.UnicodeGetString(_json, _index, c); _index += c; return s; } private string ParseName() { byte c = _json[_index++]; string s = Reflection.UTF8GetString(_json, _index, c); _index += c; return s; } private List ParseArray() { List array = new List(); bool breakparse = false; while (!breakparse) { object o = ParseValue(out breakparse); byte t = 0; if (breakparse == false) { array.Add(o); t = GetToken(); } else t = (byte)o; if (t == TOKENS.COMMA) continue; if (t == TOKENS.ARRAY_END) break; } return array; } private object ParseValue(out bool breakparse) { byte t = GetToken(); breakparse = false; switch (t) { case TOKENS.BYTE: return ParseByte(); case TOKENS.BYTEARRAY: return ParseByteArray(); case TOKENS.CHAR: return ParseChar(); case TOKENS.DATETIME: return ParseDateTime(); case TOKENS.DECIMAL: return ParseDecimal(); case TOKENS.DOUBLE: return ParseDouble(); case TOKENS.FLOAT: return ParseFloat(); case TOKENS.GUID: return ParseGuid(); case TOKENS.INT: return ParseInt(); case TOKENS.LONG: return ParseLong(); case TOKENS.SHORT: return ParseShort(); case TOKENS.UINT: return ParseUint(); case TOKENS.ULONG: return ParseULong(); case TOKENS.USHORT: return ParseUShort(); case TOKENS.UNICODE_STRING: return ParseUnicodeString(); case TOKENS.STRING: return ParseString(); case TOKENS.DOC_START: return ParseObject(); case TOKENS.ARRAY_START: return ParseArray(); case TOKENS.TRUE: return true; case TOKENS.FALSE: return false; case TOKENS.NULL: return null; case TOKENS.ARRAY_END: breakparse = true; return TOKENS.ARRAY_END; case TOKENS.DOC_END: breakparse = true; return TOKENS.DOC_END; case TOKENS.COMMA: breakparse = true; return TOKENS.COMMA; case TOKENS.ARRAY_TYPED: case TOKENS.ARRAY_TYPED_LONG: return ParseTypedArray(t); case TOKENS.TIMESPAN: return ParsTimeSpan(); } throw new Exception("Unrecognized token at index = " + _index); } private TimeSpan ParsTimeSpan() { long l = Helper.ToInt64(_json, _index); _index += 8; TimeSpan dt = new TimeSpan(l); return dt; } private object ParseTypedArray(byte token) { typedarray ar = new typedarray(); if (token == TOKENS.ARRAY_TYPED) { if (_v1_4TA) ar.typename = ParseName(); else ar.typename = ParseName2(); } else ar.typename = ParseNameLong(); ar.count = ParseInt(); bool breakparse = false; while (!breakparse) { object o = ParseValue(out breakparse); byte b = 0; if (breakparse == false) { ar.data.Add(o); b = GetToken(); } else b = (byte)o; if (b == TOKENS.COMMA) continue; if (b == TOKENS.ARRAY_END) break; } return ar; } private string ParseNameLong() // unicode short len string -> <32k chars { short c = Helper.ToInt16(_json, _index); _index += 2; string s = Reflection.UnicodeGetString(_json, _index, c); _index += c; return s; } private object ParseChar() { short u = Helper.ToInt16(_json, _index); _index += 2; return u; } private Guid ParseGuid() { byte[] b = new byte[16]; Buffer.BlockCopy(_json, _index, b, 0, 16); _index += 16; return new Guid(b); } private float ParseFloat() { float f = BitConverter.ToSingle(_json, _index); _index += 4; return f; } private ushort ParseUShort() { ushort u = (ushort)Helper.ToInt16(_json, _index); _index += 2; return u; } private ulong ParseULong() { ulong u = (ulong)Helper.ToInt64(_json, _index); _index += 8; return u; } private uint ParseUint() { uint u = (uint)Helper.ToInt32(_json, _index); _index += 4; return u; } private short ParseShort() { short u = (short)Helper.ToInt16(_json, _index); _index += 2; return u; } private long ParseLong() { long u = (long)Helper.ToInt64(_json, _index); _index += 8; return u; } private int ParseInt() { int u = (int)Helper.ToInt32(_json, _index); _index += 4; return u; } private double ParseDouble() { double d = BitConverter.ToDouble(_json, _index); _index += 8; return d; } private object ParseUnicodeString() { int c = Helper.ToInt32(_json, _index); _index += 4; string s = Reflection.UnicodeGetString(_json, _index, c); _index += c; return s; } private string ParseString() { int c = Helper.ToInt32(_json, _index); _index += 4; string s = Reflection.UTF8GetString(_json, _index, c); _index += c; return s; } private decimal ParseDecimal() { int[] i = new int[4]; i[0] = Helper.ToInt32(_json, _index); _index += 4; i[1] = Helper.ToInt32(_json, _index); _index += 4; i[2] = Helper.ToInt32(_json, _index); _index += 4; i[3] = Helper.ToInt32(_json, _index); _index += 4; return new decimal(i); } private DateTime ParseDateTime() { long l = Helper.ToInt64(_json, _index); _index += 8; DateTime dt = new DateTime(l); if (_useUTC) dt = dt.ToLocalTime(); // to local time return dt; } private byte[] ParseByteArray() { int c = Helper.ToInt32(_json, _index); _index += 4; byte[] b = new byte[c]; Buffer.BlockCopy(_json, _index, b, 0, c); _index += c; return b; } private byte ParseByte() { return _json[_index++]; } private byte GetToken() { byte b = _json[_index++]; return b; } } } ================================================ FILE: RaptorDB.Common/fastBinaryJSON/BJsonSerializer.cs ================================================ using System; using System.Collections; using System.Collections.Generic; #if !SILVERLIGHT using System.Data; #endif using System.IO; using System.Collections.Specialized; using fastJSON; namespace fastBinaryJSON { internal sealed class BJSONSerializer : IDisposable { private MemoryStream _output = new MemoryStream(); //private MemoryStream _before = new MemoryStream(); private int _typespointer = 0; private int _MAX_DEPTH = 20; int _current_depth = 0; private Dictionary _globalTypes = new Dictionary(); private Dictionary _cirobj = new Dictionary(); private BJSONParameters _params; private void Dispose(bool disposing) { if (disposing) { // dispose managed resources _output.Close(); //_before.Close(); } // free native resources } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } internal BJSONSerializer(BJSONParameters param) { _params = param; _MAX_DEPTH = param.SerializerMaxDepth; } internal byte[] ConvertToBJSON(object obj) { WriteValue(obj); // add $types if (_params.UsingGlobalTypes && _globalTypes != null && _globalTypes.Count > 0) { var pointer = (int)_output.Length; WriteName("$types"); WriteColon(); WriteTypes(_globalTypes); //var i = _output.Length; _output.Seek(_typespointer, SeekOrigin.Begin); _output.Write(Helper.GetBytes(pointer, false), 0, 4); return _output.ToArray(); } return _output.ToArray(); } private void WriteTypes(Dictionary dic) { _output.WriteByte(TOKENS.DOC_START); bool pendingSeparator = false; foreach (var entry in dic) { if (pendingSeparator) WriteComma(); WritePair(entry.Value.ToString(), entry.Key); pendingSeparator = true; } _output.WriteByte(TOKENS.DOC_END); } private void WriteValue(object obj) { if (obj == null || obj is DBNull) WriteNull(); else if (obj is string) WriteString((string)obj); else if (obj is char) WriteChar((char)obj); else if (obj is Guid) WriteGuid((Guid)obj); else if (obj is bool) WriteBool((bool)obj); else if (obj is int) WriteInt((int)obj); else if (obj is uint) WriteUInt((uint)obj); else if (obj is long) WriteLong((long)obj); else if (obj is ulong) WriteULong((ulong)obj); else if (obj is decimal) WriteDecimal((decimal)obj); else if (obj is sbyte) WriteSByte((sbyte)obj); else if (obj is byte) WriteByte((byte)obj); else if (obj is double) WriteDouble((double)obj); else if (obj is float) WriteFloat((float)obj); else if (obj is short) WriteShort((short)obj); else if (obj is ushort) WriteUShort((ushort)obj); else if (obj is DateTime) WriteDateTime((DateTime)obj); else if (obj is TimeSpan) WriteTimeSpan((TimeSpan)obj); #if NET4 else if (obj is System.Dynamic.ExpandoObject) WriteStringDictionary((IDictionary)obj); #endif else if (obj is IDictionary && obj.GetType().IsGenericType && obj.GetType().GetGenericArguments()[0] == typeof(string)) WriteStringDictionary((IDictionary)obj); else if (obj is IDictionary) WriteDictionary((IDictionary)obj); #if !SILVERLIGHT else if (obj is DataSet) WriteDataset((DataSet)obj); else if (obj is DataTable) WriteDataTable((DataTable)obj); #endif else if (obj is byte[]) WriteBytes((byte[])obj); else if (obj is StringDictionary) WriteSD((StringDictionary)obj); else if (obj is NameValueCollection) WriteNV((NameValueCollection)obj); else if (_params.UseTypedArrays && obj is Array) WriteTypedArray((ICollection)obj); else if (obj is IEnumerable) WriteArray((IEnumerable)obj); else if (obj is Enum) WriteEnum((Enum)obj); else if (Reflection.Instance.IsTypeRegistered(obj.GetType())) WriteCustom(obj); else WriteObject(obj); } private void WriteSByte(sbyte p) { _output.WriteByte(TOKENS.BYTE); byte i = (byte)p; _output.WriteByte(i); } private void WriteTimeSpan(TimeSpan obj) { _output.WriteByte(TOKENS.TIMESPAN); byte[] b = Helper.GetBytes(obj.Ticks, false); _output.Write(b, 0, b.Length); } private void WriteTypedArray(ICollection array) { bool pendingSeperator = false; bool token = true; var t = array.GetType(); if (t.IsGenericType == false)// != null) // non generic array { //if (t.GetElementType().IsClass) { token = false; byte[] b; // array type name if (_params.v1_4TypedArray) b = Reflection.UTF8GetBytes(Reflection.Instance.GetTypeAssemblyName(t.GetElementType())); else b = Reflection.UnicodeGetBytes(Reflection.Instance.GetTypeAssemblyName(t.GetElementType())); if (b.Length < 256) { _output.WriteByte(TOKENS.ARRAY_TYPED); _output.WriteByte((byte)b.Length); _output.Write(b, 0, b.Length); } else { _output.WriteByte(TOKENS.ARRAY_TYPED_LONG); _output.Write(Helper.GetBytes(b.Length, false), 0, 2); _output.Write(b, 0, b.Length); } // array count _output.Write(Helper.GetBytes(array.Count, false), 0, 4); //count } } if (token) _output.WriteByte(TOKENS.ARRAY_START); foreach (object obj in array) { if (pendingSeperator) WriteComma(); WriteValue(obj); pendingSeperator = true; } _output.WriteByte(TOKENS.ARRAY_END); } private void WriteNV(NameValueCollection nameValueCollection) { _output.WriteByte(TOKENS.DOC_START); bool pendingSeparator = false; foreach (string key in nameValueCollection) { if (pendingSeparator) _output.WriteByte(TOKENS.COMMA); WritePair(key, nameValueCollection[key]); pendingSeparator = true; } _output.WriteByte(TOKENS.DOC_END); } private void WriteSD(StringDictionary stringDictionary) { _output.WriteByte(TOKENS.DOC_START); bool pendingSeparator = false; foreach (DictionaryEntry entry in stringDictionary) { if (pendingSeparator) _output.WriteByte(TOKENS.COMMA); WritePair((string)entry.Key, entry.Value); pendingSeparator = true; } _output.WriteByte(TOKENS.DOC_END); } private void WriteUShort(ushort p) { _output.WriteByte(TOKENS.USHORT); _output.Write(Helper.GetBytes(p, false), 0, 2); } private void WriteShort(short p) { _output.WriteByte(TOKENS.SHORT); _output.Write(Helper.GetBytes(p, false), 0, 2); } private void WriteFloat(float p) { _output.WriteByte(TOKENS.FLOAT); byte[] b = BitConverter.GetBytes(p); _output.Write(b, 0, b.Length); } private void WriteDouble(double p) { _output.WriteByte(TOKENS.DOUBLE); var b = BitConverter.GetBytes(p); _output.Write(b, 0, b.Length); } private void WriteByte(byte p) { _output.WriteByte(TOKENS.BYTE); _output.WriteByte(p); } private void WriteDecimal(decimal p) { _output.WriteByte(TOKENS.DECIMAL); var b = decimal.GetBits(p); foreach (var c in b) _output.Write(Helper.GetBytes(c, false), 0, 4); } private void WriteULong(ulong p) { _output.WriteByte(TOKENS.ULONG); _output.Write(Helper.GetBytes((long)p, false), 0, 8); } private void WriteUInt(uint p) { _output.WriteByte(TOKENS.UINT); _output.Write(Helper.GetBytes(p, false), 0, 4); } private void WriteLong(long p) { _output.WriteByte(TOKENS.LONG); _output.Write(Helper.GetBytes(p, false), 0, 8); } private void WriteChar(char p) { _output.WriteByte(TOKENS.CHAR); _output.Write(Helper.GetBytes((short)p, false), 0, 2); } private void WriteBytes(byte[] p) { _output.WriteByte(TOKENS.BYTEARRAY); _output.Write(Helper.GetBytes(p.Length, false), 0, 4); _output.Write(p, 0, p.Length); } private void WriteBool(bool p) { if (p) _output.WriteByte(TOKENS.TRUE); else _output.WriteByte(TOKENS.FALSE); } private void WriteNull() { _output.WriteByte(TOKENS.NULL); } private void WriteCustom(object obj) { Reflection.Serialize s; Reflection.Instance._customSerializer.TryGetValue(obj.GetType(), out s); WriteString(s(obj)); } private void WriteColon() { _output.WriteByte(TOKENS.COLON); } private void WriteComma() { _output.WriteByte(TOKENS.COMMA); } private void WriteEnum(Enum e) { WriteString(e.ToString()); } private void WriteInt(int i) { _output.WriteByte(TOKENS.INT); _output.Write(Helper.GetBytes(i, false), 0, 4); } private void WriteGuid(Guid g) { _output.WriteByte(TOKENS.GUID); _output.Write(g.ToByteArray(), 0, 16); } private void WriteDateTime(DateTime dateTime) { DateTime dt = dateTime; if (_params.UseUTCDateTime) dt = dateTime.ToUniversalTime(); _output.WriteByte(TOKENS.DATETIME); byte[] b = Helper.GetBytes(dt.Ticks, false); _output.Write(b, 0, b.Length); } #if !SILVERLIGHT private DatasetSchema GetSchema(DataTable ds) { if (ds == null) return null; DatasetSchema m = new DatasetSchema(); m.Info = new List(); m.Name = ds.TableName; foreach (DataColumn c in ds.Columns) { m.Info.Add(ds.TableName); m.Info.Add(c.ColumnName); m.Info.Add(c.DataType.ToString()); } // FEATURE : serialize relations and constraints here return m; } private DatasetSchema GetSchema(DataSet ds) { if (ds == null) return null; DatasetSchema m = new DatasetSchema(); m.Info = new List(); m.Name = ds.DataSetName; foreach (DataTable t in ds.Tables) { foreach (DataColumn c in t.Columns) { m.Info.Add(t.TableName); m.Info.Add(c.ColumnName); m.Info.Add(c.DataType.ToString()); } } // FEATURE : serialize relations and constraints here return m; } private string GetXmlSchema(DataTable dt) { using (var writer = new StringWriter()) { dt.WriteXmlSchema(writer); return dt.ToString(); } } private void WriteDataset(DataSet ds) { _output.WriteByte(TOKENS.DOC_START); { WritePair("$schema", _params.UseOptimizedDatasetSchema ? (object)GetSchema(ds) : ds.GetXmlSchema()); WriteComma(); } bool tablesep = false; foreach (DataTable table in ds.Tables) { if (tablesep) WriteComma(); tablesep = true; WriteDataTableData(table); } // end dataset _output.WriteByte(TOKENS.DOC_END); } private void WriteDataTableData(DataTable table) { WriteName(table.TableName); WriteColon(); _output.WriteByte(TOKENS.ARRAY_START); DataColumnCollection cols = table.Columns; bool rowseparator = false; foreach (DataRow row in table.Rows) { if (rowseparator) WriteComma(); rowseparator = true; _output.WriteByte(TOKENS.ARRAY_START); bool pendingSeperator = false; foreach (DataColumn column in cols) { if (pendingSeperator) WriteComma(); WriteValue(row[column]); pendingSeperator = true; } _output.WriteByte(TOKENS.ARRAY_END); } _output.WriteByte(TOKENS.ARRAY_END); } void WriteDataTable(DataTable dt) { _output.WriteByte(TOKENS.DOC_START); //if (this.useExtension) { this.WritePair("$schema", _params.UseOptimizedDatasetSchema ? (object)this.GetSchema(dt) : this.GetXmlSchema(dt)); WriteComma(); } WriteDataTableData(dt); // end datatable _output.WriteByte(TOKENS.DOC_END); } #endif bool _TypesWritten = false; private void WriteObject(object obj) { int i = 0; if (_cirobj.TryGetValue(obj, out i) == false) _cirobj.Add(obj, _cirobj.Count + 1); else { if (_current_depth > 0) { //_circular = true; _output.WriteByte(TOKENS.DOC_START); WriteName("$i"); WriteColon(); WriteValue(i); _output.WriteByte(TOKENS.DOC_END); return; } } if (_params.UsingGlobalTypes == false) _output.WriteByte(TOKENS.DOC_START); else { if (_TypesWritten == false) { _output.WriteByte(TOKENS.DOC_START); // write pointer to $types position _output.WriteByte(TOKENS.TYPES_POINTER); _typespointer = (int)_output.Length; // place holder _output.Write(new byte[4], 0, 4); // zero pointer for now //_output = new MemoryStream(); _TypesWritten = true; } else _output.WriteByte(TOKENS.DOC_START); } _current_depth++; if (_current_depth > _MAX_DEPTH) throw new Exception("Serializer encountered maximum depth of " + _MAX_DEPTH); Type t = obj.GetType(); bool append = false; if (_params.UseExtensions) { if (_params.UsingGlobalTypes == false) WritePairFast("$type", Reflection.Instance.GetTypeAssemblyName(t)); else { int dt = 0; string ct = Reflection.Instance.GetTypeAssemblyName(t); if (_globalTypes.TryGetValue(ct, out dt) == false) { dt = _globalTypes.Count + 1; _globalTypes.Add(ct, dt); } WritePairFast("$type", dt.ToString()); } append = true; } Getters[] g = Reflection.Instance.GetGetters(t, /*_params.ShowReadOnlyProperties,*/ _params.IgnoreAttributes); int c = g.Length; for (int ii = 0; ii < c; ii++) { var p = g[ii]; var o = p.Getter(obj); if (_params.SerializeNulls == false && (o == null || o is DBNull)) { } else { if (append) WriteComma(); WritePair(p.Name, o); append = true; } } _output.WriteByte(TOKENS.DOC_END); _current_depth--; } private void WritePairFast(string name, string value) { if (_params.SerializeNulls == false && (value == null)) return; WriteName(name); WriteColon(); WriteString(value); } private void WritePair(string name, object value) { if (_params.SerializeNulls == false && (value == null || value is DBNull)) return; WriteName(name); WriteColon(); WriteValue(value); } private void WriteArray(IEnumerable array) { _output.WriteByte(TOKENS.ARRAY_START); bool pendingSeperator = false; foreach (object obj in array) { if (pendingSeperator) WriteComma(); WriteValue(obj); pendingSeperator = true; } _output.WriteByte(TOKENS.ARRAY_END); } private void WriteStringDictionary(IDictionary dic) { _output.WriteByte(TOKENS.DOC_START); bool pendingSeparator = false; foreach (DictionaryEntry entry in dic) { if (pendingSeparator) WriteComma(); WritePair((string)entry.Key, entry.Value); pendingSeparator = true; } _output.WriteByte(TOKENS.DOC_END); } private void WriteStringDictionary(IDictionary dic) { _output.WriteByte(TOKENS.DOC_START); bool pendingSeparator = false; foreach (KeyValuePair entry in dic) { if (pendingSeparator) WriteComma(); WritePair((string)entry.Key, entry.Value); pendingSeparator = true; } _output.WriteByte(TOKENS.DOC_END); } private void WriteDictionary(IDictionary dic) { _output.WriteByte(TOKENS.ARRAY_START); bool pendingSeparator = false; foreach (DictionaryEntry entry in dic) { if (pendingSeparator) WriteComma(); _output.WriteByte(TOKENS.DOC_START); WritePair("k", entry.Key); WriteComma(); WritePair("v", entry.Value); _output.WriteByte(TOKENS.DOC_END); pendingSeparator = true; } _output.WriteByte(TOKENS.ARRAY_END); } private void WriteName(string s) { byte[] b; if (_params.UseUnicodeStrings == false) { _output.WriteByte(TOKENS.NAME); b = Reflection.UTF8GetBytes(s); } else { _output.WriteByte(TOKENS.NAME_UNI); b = Reflection.UnicodeGetBytes(s); } _output.WriteByte((byte)b.Length); _output.Write(b, 0, b.Length % 256); } private void WriteString(string s) { byte[] b = null; if (_params.UseUnicodeStrings) { _output.WriteByte(TOKENS.UNICODE_STRING); b = Reflection.UnicodeGetBytes(s); } else { _output.WriteByte(TOKENS.STRING); b = Reflection.UTF8GetBytes(s); } _output.Write(Helper.GetBytes(b.Length, false), 0, 4); _output.Write(b, 0, b.Length); } } } ================================================ FILE: RaptorDB.Common/fastBinaryJSON/Helper.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Text; namespace fastBinaryJSON { internal static class Helper { internal static unsafe int ToInt32(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[4]; Buffer.BlockCopy(value, startIndex, b, 0, 4); Array.Reverse(b); return ToInt32(b, 0); } return ToInt32(value, startIndex); } internal static unsafe int ToInt32(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *((int*)numRef); } } internal static unsafe long ToInt64(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[8]; Buffer.BlockCopy(value, startIndex, b, 0, 8); Array.Reverse(b); return ToInt64(b, 0); } return ToInt64(value, startIndex); } internal static unsafe long ToInt64(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((long*)numRef)); } } internal static unsafe short ToInt16(byte[] value, int startIndex, bool reverse) { if (reverse) { byte[] b = new byte[2]; Buffer.BlockCopy(value, startIndex, b, 0, 2); Array.Reverse(b); return ToInt16(b, 0); } return ToInt16(value, startIndex); } internal static unsafe short ToInt16(byte[] value, int startIndex) { fixed (byte* numRef = &(value[startIndex])) { return *(((short*)numRef)); } } internal static unsafe byte[] GetBytes(long num, bool reverse) { byte[] buffer = new byte[8]; fixed (byte* numRef = buffer) { *((long*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } public static unsafe byte[] GetBytes(int num, bool reverse) { byte[] buffer = new byte[4]; fixed (byte* numRef = buffer) { *((int*)numRef) = num; } if (reverse) Array.Reverse(buffer); return buffer; } } } ================================================ FILE: RaptorDB.Common/fastBinaryJSON/dynamic.cs ================================================ #if NET4 using System; using System.Collections; using System.Collections.Generic; using System.Dynamic; namespace fastBinaryJSON { internal class DynamicJson : DynamicObject, IEnumerable { private IDictionary _dictionary { get; set; } private List _list { get; set; } public DynamicJson(byte[] json) { var parse = fastBinaryJSON.BJSON.Parse(json); if (parse is IDictionary) _dictionary = (IDictionary)parse; else if (parse is typedarray) _list = ((typedarray)parse).data; else _list = (List)parse; } private DynamicJson(object dictionary) { if (dictionary is IDictionary) _dictionary = (IDictionary)dictionary; } public override bool TryGetIndex(GetIndexBinder binder, Object[] indexes, out Object result) { var index = indexes[0]; if (index is int) { result = _list[(int)index]; } else { result = _dictionary[(string)index]; } if (result is IDictionary) result = new DynamicJson(result as IDictionary); return true; } public override bool TryGetMember(GetMemberBinder binder, out object result) { if (_dictionary.TryGetValue(binder.Name, out result) == false) if (_dictionary.TryGetValue(binder.Name.ToLowerInvariant(), out result) == false) return false;// throw new Exception("property not found " + binder.Name); if (result is IDictionary) { result = new DynamicJson(result as IDictionary); } else if (result is List) { List list = new List(); foreach (object item in (List)result) { if (item is IDictionary) list.Add(new DynamicJson(item as IDictionary)); else list.Add(item); } result = list; } return _dictionary.ContainsKey(binder.Name); } public IEnumerator GetEnumerator() { foreach (var o in _list) { yield return new DynamicJson(o as IDictionary); } } } } #endif ================================================ FILE: RaptorDB.Common/fastJSON/Formatter.cs ================================================ using System.Text; namespace fastJSON { internal static class Formatter { //private static string _indent = " "; private static void AppendIndent(StringBuilder sb, int count, string indent) { for (; count > 0; --count) sb.Append(indent); } public static string PrettyPrint(string input) { return PrettyPrint(input, new string(' ', JSON.Parameters.FormatterIndentSpaces));// " "); } public static string PrettyPrint(string input, string spaces) { //_indent = spaces; var output = new StringBuilder(); int depth = 0; int len = input.Length; char[] chars = input.ToCharArray(); for (int i = 0; i < len; ++i) { char ch = chars[i]; if (ch == '\"') // found string span { bool str = true; while (str) { output.Append(ch); ch = chars[++i]; if (ch == '\\') { output.Append(ch); ch = chars[++i]; } else if (ch == '\"') str = false; } } switch (ch) { case '{': case '[': output.Append(ch); output.AppendLine(); AppendIndent(output, ++depth, spaces); break; case '}': case ']': output.AppendLine(); AppendIndent(output, --depth, spaces); output.Append(ch); break; case ',': output.Append(ch); output.AppendLine(); AppendIndent(output, depth, spaces); break; case ':': output.Append(" : "); break; default: if (!char.IsWhiteSpace(ch)) output.Append(ch); break; } } return output.ToString(); } } } ================================================ FILE: RaptorDB.Common/fastJSON/Getters.cs ================================================ using System.Collections.Generic; namespace fastJSON { public sealed class DatasetSchema { public List Info ;//{ get; set; } public string Name ;//{ get; set; } } } ================================================ FILE: RaptorDB.Common/fastJSON/Helper.cs ================================================ using System; using System.Collections.Generic; using System.Collections.Specialized; namespace fastJSON { //public class FJObject : Dictionary //{ //} class Helper { public static bool IsNullable(Type t) { if (!t.IsGenericType) return false; Type g = t.GetGenericTypeDefinition(); return (g.Equals(typeof(Nullable<>))); } public static Type UnderlyingTypeOf(Type t) { return Reflection.Instance.GetGenericArguments(t)[0]; } public static DateTimeOffset CreateDateTimeOffset(int year, int month, int day, int hour, int min, int sec, int milli, int extraTicks, TimeSpan offset) { var dt = new DateTimeOffset(year, month, day, hour, min, sec, milli, offset); if (extraTicks > 0) dt += TimeSpan.FromTicks(extraTicks); return dt; } public static bool BoolConv(object v) { bool oset = false; if (v is bool) oset = (bool)v; else if (v is long) oset = (long)v > 0 ? true : false; else if (v is string) { var s = (string)v; s = s.ToLowerInvariant(); if (s == "1" || s == "true" || s == "yes" || s == "on") oset = true; } return oset; } public static long AutoConv(object value, JSONParameters param) { if (value is string) { if (param.AutoConvertStringToNumbers == true) { string s = (string)value; return CreateLong(s, 0, s.Length); } else throw new Exception("AutoConvertStringToNumbers is disabled for converting string : " + value); } else if (value is long) return (long)value; else return Convert.ToInt64(value); } public static long CreateLong(string s, int index, int count) { //long num = 0; //bool neg = false; //for (int x = 0; x < count; x++, index++) //{ // char cc = s[index]; // if (cc == '-') // neg = true; // else if (cc == '+') // neg = false; // else // { // num *= 10; // num += (int)(cc - '0'); // } //} //if (neg) num = -num; //return num; bool neg = false; char cc = s[index++]; if (cc == '-') { neg = true; count--; cc = s[index++]; } else if (cc == '+') { count--; cc = s[index++]; } long num = (long)(cc - '0'); while (--count > 0) num = num * 10 + (long)(s[index++] - '0'); if (neg) num = -num; return num; } public static long CreateLong(char[] s, int index, int count) { //long num = 0; //bool neg = false; //for (int x = 0; x < count; x++, index++) //{ // char cc = s[index]; // if (cc == '-') // neg = true; // else if (cc == '+') // neg = false; // else // { // num *= 10; // num += (int)(cc - '0'); // } //} //if (neg) num = -num; //return num; bool neg = false; char cc = s[index++]; if (cc == '-') { neg = true; count--; cc = s[index++]; } else if (cc == '+') { count--; cc = s[index++]; } long num = (long)(cc - '0'); while (--count > 0) num = num * 10 + (long)(s[index++] - '0'); if (neg) num = -num; return num; } public static int CreateInteger(string s, int index, int count) { //int num = 0; //bool neg = false; //for (int x = 0; x < count; x++, index++) //{ // char cc = s[index]; // if (cc == '-') // neg = true; // else if (cc == '+') // neg = false; // else // { // num *= 10; // num += (int)(cc - '0'); // } //} //if (neg) num = -num; //return num; bool neg = false; char cc = s[index++]; if (cc == '-') { neg = true; count--; cc = s[index++]; } else if (cc == '+') { count--; cc = s[index++]; } int num = (int)(cc - '0'); while (--count > 0) num = num * 10 + (int)(s[index++] - '0'); if (neg) num = -num; return num; } public static object CreateEnum(Type pt, object v) { // FEATURE : optimize create enum #if !SILVERLIGHT return Enum.Parse(pt, v.ToString(), true); #else return Enum.Parse(pt, v, true); #endif } public static Guid CreateGuid(string s) { if (s.Length > 30) return new Guid(s); else return new Guid(Convert.FromBase64String(s)); } public static StringDictionary CreateSD(Dictionary d) { StringDictionary nv = new StringDictionary(); foreach (var o in d) nv.Add(o.Key, (string)o.Value); return nv; } public static NameValueCollection CreateNV(Dictionary d) { NameValueCollection nv = new NameValueCollection(); foreach (var o in d) nv.Add(o.Key, (string)o.Value); return nv; } public static object CreateDateTimeOffset(string value) { // 0123456789012345678 9012 9/3 0/4 1/5 // datetime format = yyyy-MM-ddTHH:mm:ss .nnn _ + 00:00 // ISO8601 roundtrip formats have 7 digits for ticks, and no space before the '+' // datetime format = yyyy-MM-ddTHH:mm:ss .nnnnnnn + 00:00 // datetime format = yyyy-MM-ddTHH:mm:ss .nnnnnnn Z int year; int month; int day; int hour; int min; int sec; int ms = 0; int usTicks = 0; // ticks for xxx.x microseconds int th = 0; int tm = 0; year = CreateInteger(value, 0, 4); month = CreateInteger(value, 5, 2); day = CreateInteger(value, 8, 2); hour = CreateInteger(value, 11, 2); min = CreateInteger(value, 14, 2); sec = CreateInteger(value, 17, 2); int p = 20; if (value.Length > 21 && value[19] == '.') { ms = CreateInteger(value, p, 3); p = 23; // handle 7 digit case if (value.Length > 25 && char.IsDigit(value[p])) { usTicks = CreateInteger(value, p, 4); p = 27; } } if (value[p] == 'Z') // UTC return CreateDateTimeOffset(year, month, day, hour, min, sec, ms, usTicks, TimeSpan.Zero); if (value[p] == ' ') ++p; // +00:00 th = CreateInteger(value, p + 1, 2); tm = CreateInteger(value, p + 1 + 2 + 1, 2); if (value[p] == '-') th = -th; return CreateDateTimeOffset(year, month, day, hour, min, sec, ms, usTicks, new TimeSpan(th, tm, 0)); } public static DateTime CreateDateTime(string value, bool UseUTCDateTime) { if (value.Length < 19) return DateTime.MinValue; bool utc = false; // 0123456789012345678 9012 9/3 // datetime format = yyyy-MM-ddTHH:mm:ss .nnn Z int year; int month; int day; int hour; int min; int sec; int ms = 0; year = CreateInteger(value, 0, 4); month = CreateInteger(value, 5, 2); day = CreateInteger(value, 8, 2); hour = CreateInteger(value, 11, 2); min = CreateInteger(value, 14, 2); sec = CreateInteger(value, 17, 2); if (value.Length > 21 && value[19] == '.') ms = CreateInteger(value, 20, 3); if (value[value.Length - 1] == 'Z') utc = true; if (UseUTCDateTime == false && utc == false) return new DateTime(year, month, day, hour, min, sec, ms); else return new DateTime(year, month, day, hour, min, sec, ms, DateTimeKind.Utc).ToLocalTime(); } } } ================================================ FILE: RaptorDB.Common/fastJSON/JSON.cs ================================================ using System; using System.Collections; using System.Collections.Generic; #if !SILVERLIGHT using System.Data; #endif using System.Globalization; using System.IO; using System.Collections.Specialized; using RaptorDB.Common; namespace fastJSON { //public delegate string Serialize(object data); //public delegate object Deserialize(string data); public sealed class JSONParameters { /// /// Use the optimized fast Dataset Schema format (default = True) /// public bool UseOptimizedDatasetSchema = true; /// /// Use the fast GUID format (default = True) /// public bool UseFastGuid = true; /// /// Serialize null values to the output (default = True) /// public bool SerializeNullValues = true; /// /// Use the UTC date format (default = True) /// public bool UseUTCDateTime = true; /// /// Show the readonly properties of types in the output (default = False) /// public bool ShowReadOnlyProperties = false; /// /// Use the $types extension to optimise the output json (default = True) /// public bool UsingGlobalTypes = true; /// /// Ignore case when processing json and deserializing /// [Obsolete("Not needed anymore and will always match")] public bool IgnoreCaseOnDeserialize = false; /// /// Anonymous types have read only properties /// public bool EnableAnonymousTypes = false; /// /// Enable fastJSON extensions $types, $type, $map (default = True) /// public bool UseExtensions = true; /// /// Use escaped unicode i.e. \uXXXX format for non ASCII characters (default = True) /// public bool UseEscapedUnicode = true; /// /// Output string key dictionaries as "k"/"v" format (default = False) /// public bool KVStyleStringDictionary = false; /// /// Output Enum values instead of names (default = False) /// public bool UseValuesOfEnums = false; /// /// Ignore attributes to check for (default : XmlIgnoreAttribute, NonSerialized) /// public List IgnoreAttributes = new List { typeof(System.Xml.Serialization.XmlIgnoreAttribute), typeof(NonSerializedAttribute) }; /// /// If you have parametric and no default constructor for you classes (default = False) /// /// IMPORTANT NOTE : If True then all initial values within the class will be ignored and will be not set /// public bool ParametricConstructorOverride = false; /// /// Serialize DateTime milliseconds i.e. yyyy-MM-dd HH:mm:ss.nnn (default = false) /// public bool DateTimeMilliseconds = false; /// /// Maximum depth for circular references in inline mode (default = 20) /// public byte SerializerMaxDepth = 20; /// /// Inline circular or already seen objects instead of replacement with $i (default = false) /// public bool InlineCircularReferences = false; /// /// Save property/field names as lowercase (default = false) /// public bool SerializeToLowerCaseNames = false; /// /// Formatter indent spaces (default = 3) /// public byte FormatterIndentSpaces = 3; /// /// TESTING - allow non quoted keys in the json like javascript (default = false) /// public bool AllowNonQuotedKeys = false; /// /// Auto convert string values to numbers when needed (default = true) /// /// When disabled you will get an exception if the types don't match /// public bool AutoConvertStringToNumbers = true; /// /// Override object equality hash code checking (default = false) /// public bool OverrideObjectHashCodeChecking = false; /// /// Checking black list types to prevent friday 13th json attacks (default = true) /// /// Will throw an exception if encountered and set /// public bool BlackListTypeChecking = true; public void FixValues() { if (UseExtensions == false) // disable conflicting params { UsingGlobalTypes = false; InlineCircularReferences = true; } if (EnableAnonymousTypes) ShowReadOnlyProperties = true; } public JSONParameters MakeCopy() { return new JSONParameters { AllowNonQuotedKeys = AllowNonQuotedKeys, DateTimeMilliseconds = DateTimeMilliseconds, EnableAnonymousTypes = EnableAnonymousTypes, FormatterIndentSpaces = FormatterIndentSpaces, IgnoreAttributes = new List(IgnoreAttributes), //IgnoreCaseOnDeserialize = IgnoreCaseOnDeserialize, InlineCircularReferences = InlineCircularReferences, KVStyleStringDictionary = KVStyleStringDictionary, ParametricConstructorOverride = ParametricConstructorOverride, SerializeNullValues = SerializeNullValues, SerializerMaxDepth = SerializerMaxDepth, SerializeToLowerCaseNames = SerializeToLowerCaseNames, ShowReadOnlyProperties = ShowReadOnlyProperties, UseEscapedUnicode = UseEscapedUnicode, UseExtensions = UseExtensions, UseFastGuid = UseFastGuid, UseOptimizedDatasetSchema = UseOptimizedDatasetSchema, UseUTCDateTime = UseUTCDateTime, UseValuesOfEnums = UseValuesOfEnums, UsingGlobalTypes = UsingGlobalTypes, AutoConvertStringToNumbers = AutoConvertStringToNumbers, OverrideObjectHashCodeChecking = OverrideObjectHashCodeChecking, BlackListTypeChecking = BlackListTypeChecking }; } } public static class JSON { /// /// Globally set-able parameters for controlling the serializer /// public static JSONParameters Parameters = new JSONParameters(); /// /// Create a formatted json string (beautified) from an object /// /// /// public static string ToNiceJSON(object obj) { string s = ToJSON(obj, Parameters); // use default params return Beautify(s); } /// /// Create a formatted json string (beautified) from an object /// /// /// /// public static string ToNiceJSON(object obj, JSONParameters param) { string s = ToJSON(obj, param); return Beautify(s, param.FormatterIndentSpaces); } /// /// Create a json representation for an object /// /// /// public static string ToJSON(object obj) { return ToJSON(obj, Parameters); } /// /// Create a json representation for an object with parameter override on this call /// /// /// /// public static string ToJSON(object obj, JSONParameters param) { param.FixValues(); param = param.MakeCopy(); Type t = null; if (obj == null) return "null"; if (obj.GetType().IsGenericType) t = Reflection.Instance.GetGenericTypeDefinition(obj.GetType()); if (t == typeof(Dictionary<,>) || t == typeof(List<>)) param.UsingGlobalTypes = false; // FEATURE : enable extensions when you can deserialize anon types if (param.EnableAnonymousTypes) { param.UseExtensions = false; param.UsingGlobalTypes = false; } return new JSONSerializer(param).ConvertToJSON(obj); } /// /// Parse a json string and generate a Dictionary<string,object> or List<object> structure /// /// /// public static object Parse(string json) { return new JsonParser(json, Parameters.AllowNonQuotedKeys).Decode(); } #if NET4 /// /// Create a .net4 dynamic object from the json string /// /// /// public static dynamic ToDynamic(string json) { return new DynamicJson(json); } #endif /// /// Create a typed generic object from the json /// /// /// /// public static T ToObject(string json) { return new deserializer(Parameters).ToObject(json); } /// /// Create a typed generic object from the json with parameter override on this call /// /// /// /// /// public static T ToObject(string json, JSONParameters param) { return new deserializer(param).ToObject(json); } /// /// Create an object from the json /// /// /// public static object ToObject(string json) { return new deserializer(Parameters).ToObject(json, null); } /// /// Create an object from the json with parameter override on this call /// /// /// /// public static object ToObject(string json, JSONParameters param) { return new deserializer(param).ToObject(json, null); } /// /// Create an object of type from the json /// /// /// /// public static object ToObject(string json, Type type) { return new deserializer(Parameters).ToObject(json, type); } /// /// Create an object of type from the json with parameter override on this call /// /// /// /// /// public static object ToObject(string json, Type type, JSONParameters par) { return new deserializer(par).ToObject(json, type); } /// /// Fill a given object with the json represenation /// /// /// /// public static object FillObject(object input, string json) { Dictionary ht = new JsonParser(json, Parameters.AllowNonQuotedKeys).Decode() as Dictionary; if (ht == null) return null; return new deserializer(Parameters).ParseDictionary(ht, null, input.GetType(), input); } /// /// Deep copy an object i.e. clone to a new object /// /// /// public static object DeepCopy(object obj) { return new deserializer(Parameters).ToObject(ToJSON(obj)); } /// /// /// /// /// /// public static T DeepCopy(T obj) { return new deserializer(Parameters).ToObject(ToJSON(obj)); } /// /// Create a human readable string from the json /// /// /// public static string Beautify(string input) { var i = new string(' ', JSON.Parameters.FormatterIndentSpaces); return Formatter.PrettyPrint(input, i); } /// /// Create a human readable string from the json with specified indent spaces /// /// /// /// public static string Beautify(string input, byte spaces) { var i = new string(' ', spaces); return Formatter.PrettyPrint(input, i); } /// /// Register custom type handlers for your own types not natively handled by fastJSON /// /// /// /// public static void RegisterCustomType(Type type, Reflection.Serialize serializer, Reflection.Deserialize deserializer) { Reflection.Instance.RegisterCustomType(type, serializer, deserializer); } /// /// Clear the internal reflection cache so you can start from new (you will loose performance) /// public static void ClearReflectionCache() { Reflection.Instance.ClearReflectionCache(); } } internal class deserializer { public deserializer(JSONParameters param) { if (param.OverrideObjectHashCodeChecking) _circobj = new Dictionary(10, ReferenceEqualityComparer.Default); else _circobj = new Dictionary(); param.FixValues(); _params = param.MakeCopy(); } private JSONParameters _params; private bool _usingglobals = false; private Dictionary _circobj;// = new Dictionary(); private Dictionary _cirrev = new Dictionary(); public T ToObject(string json) { Type t = typeof(T); var o = ToObject(json, t); if (t.IsArray) { if ((o as ICollection).Count == 0) // edge case for "[]" -> T[] { Type tt = t.GetElementType(); object oo = Array.CreateInstance(tt, 0); return (T)oo; } else return (T)o; } else return (T)o; } public object ToObject(string json) { return ToObject(json, null); } public object ToObject(string json, Type type) { //_params.FixValues(); Type t = null; if (type != null && type.IsGenericType) t = Reflection.Instance.GetGenericTypeDefinition(type); _usingglobals = _params.UsingGlobalTypes; if (t == typeof(Dictionary<,>) || t == typeof(List<>)) _usingglobals = false; object o = new JsonParser(json, _params.AllowNonQuotedKeys).Decode(); if (o == null) return null; #if !SILVERLIGHT if (type != null) { if (type == typeof(DataSet)) return CreateDataset(o as Dictionary, null); else if (type == typeof(DataTable)) return CreateDataTable(o as Dictionary, null); } #endif if (o is IDictionary) { if (type != null && t == typeof(Dictionary<,>)) // deserialize a dictionary return RootDictionary(o, type); else // deserialize an object return ParseDictionary(o as Dictionary, null, type, null); } else if (o is List) { if (type != null) { if (t == typeof(Dictionary<,>)) // kv format return RootDictionary(o, type); else if (t == typeof(List<>)) // deserialize to generic list return RootList(o, type); else if (type.IsArray) return RootArray(o, type); else if (type == typeof(Hashtable)) return RootHashTable((List)o); } else //if (type == null) { List l = (List)o; if (l.Count > 0 && l[0].GetType() == typeof(Dictionary)) { Dictionary globals = new Dictionary(); List op = new List(); // try to get $types foreach (var i in l) op.Add(ParseDictionary((Dictionary)i, globals, null, null)); return op; } return l.ToArray(); } } else if (type != null && o.GetType() != type) return ChangeType(o, type); return o; } #region [ p r i v a t e m e t h o d s ] private object RootHashTable(List o) { Hashtable h = new Hashtable(); foreach (Dictionary values in o) { object key = values["k"]; object val = values["v"]; if (key is Dictionary) key = ParseDictionary((Dictionary)key, null, typeof(object), null); if (val is Dictionary) val = ParseDictionary((Dictionary)val, null, typeof(object), null); h.Add(key, val); } return h; } private object ChangeType(object value, Type conversionType) { if (conversionType == typeof(int)) { string s = value as string; if (s == null) return (int)((long)value); else if (_params.AutoConvertStringToNumbers) return Helper.CreateInteger(s, 0, s.Length); else throw new Exception("AutoConvertStringToNumbers is disabled for converting string : " + value); } else if (conversionType == typeof(long)) { string s = value as string; if (s == null) return (long)value; else if (_params.AutoConvertStringToNumbers) return Helper.CreateLong(s, 0, s.Length); else throw new Exception("AutoConvertStringToNumbers is disabled for converting string : " + value); } else if (conversionType == typeof(string)) return (string)value; else if (conversionType.IsEnum) return Helper.CreateEnum(conversionType, value); else if (conversionType == typeof(DateTime)) return Helper.CreateDateTime((string)value, _params.UseUTCDateTime); else if (conversionType == typeof(DateTimeOffset)) return Helper.CreateDateTimeOffset((string)value); else if (Reflection.Instance.IsTypeRegistered(conversionType)) return Reflection.Instance.CreateCustom((string)value, conversionType); // 8-30-2014 - James Brooks - Added code for nullable types. if (Helper.IsNullable(conversionType)) { if (value == null) return value; conversionType = Helper.UnderlyingTypeOf(conversionType); } // 8-30-2014 - James Brooks - Nullable Guid is a special case so it was moved after the "IsNullable" check. if (conversionType == typeof(Guid)) return Helper.CreateGuid((string)value); // 2016-04-02 - Enrico Padovani - proper conversion of byte[] back from string if (conversionType == typeof(byte[])) return Convert.FromBase64String((string)value); if (conversionType == typeof(TimeSpan)) return new TimeSpan((long)value); return Convert.ChangeType(value, conversionType, CultureInfo.InvariantCulture); } private object RootList(object parse, Type type) { Type[] gtypes = Reflection.Instance.GetGenericArguments(type); var o = (IList)Reflection.Instance.FastCreateList(type, ((IList)parse).Count); DoParseList((IList)parse, gtypes[0], o); return o; } private void DoParseList(IList parse, Type it, IList o) { Dictionary globals = new Dictionary(); foreach (var k in parse) { _usingglobals = false; object v = k; var a = k as Dictionary; if (a != null) v = ParseDictionary(a, globals, it, null); else v = ChangeType(k, it); o.Add(v); } } private object RootArray(object parse, Type type) { Type it = type.GetElementType(); var o = (IList)Reflection.Instance.FastCreateInstance(typeof(List<>).MakeGenericType(it)); DoParseList((IList)parse, it, o); var array = Array.CreateInstance(it, o.Count); o.CopyTo(array, 0); return array; } private object RootDictionary(object parse, Type type) { Type[] gtypes = Reflection.Instance.GetGenericArguments(type); Type t1 = null; Type t2 = null; bool dictionary = false; if (gtypes != null) { t1 = gtypes[0]; t2 = gtypes[1]; if (t2 != null) dictionary = t2.Name.StartsWith("Dictionary"); } var arraytype = t2.GetElementType(); if (parse is Dictionary) { IDictionary o = (IDictionary)Reflection.Instance.FastCreateInstance(type); foreach (var kv in (Dictionary)parse) { object v; object k = ChangeType(kv.Key, t1); if (dictionary) // deserialize a dictionary v = RootDictionary(kv.Value, t2); else if (kv.Value is Dictionary) v = ParseDictionary(kv.Value as Dictionary, null, t2, null); else if (t2.IsArray && t2 != typeof(byte[])) v = CreateArray((List)kv.Value, t2, arraytype, null); else if (kv.Value is IList) v = CreateGenericList((List)kv.Value, t2, t1, null); else v = ChangeType(kv.Value, t2); o.Add(k, v); } return o; } if (parse is List) return CreateDictionary(parse as List, type, gtypes, null); return null; } internal object ParseDictionary(Dictionary d, Dictionary globaltypes, Type type, object input) { object tn = ""; if (type == typeof(NameValueCollection)) return Helper.CreateNV(d); if (type == typeof(StringDictionary)) return Helper.CreateSD(d); if (d.TryGetValue("$i", out tn)) { object v = null; _cirrev.TryGetValue((int)(long)tn, out v); return v; } if (d.TryGetValue("$types", out tn)) { _usingglobals = true; if (globaltypes == null) globaltypes = new Dictionary(); foreach (var kv in (Dictionary)tn) { globaltypes.Add((string)kv.Value, kv.Key); } } if (globaltypes != null) _usingglobals = true; bool found = d.TryGetValue("$type", out tn); #if !SILVERLIGHT if (found == false && type == typeof(System.Object)) { return d; // CreateDataset(d, globaltypes); } #endif if (found) { if (_usingglobals) { object tname = ""; if (globaltypes != null && globaltypes.TryGetValue((string)tn, out tname)) tn = tname; } type = Reflection.Instance.GetTypeFromCache((string)tn, _params.BlackListTypeChecking); } if (type == null) throw new Exception("Cannot determine type : " + tn); string typename = type.FullName; object o = input; if (o == null) { if (_params.ParametricConstructorOverride) o = System.Runtime.Serialization.FormatterServices.GetUninitializedObject(type); else o = Reflection.Instance.FastCreateInstance(type); } int circount = 0; if (_circobj.TryGetValue(o, out circount) == false) { circount = _circobj.Count + 1; _circobj.Add(o, circount); _cirrev.Add(circount, o); } var props = Reflection.Instance.Getproperties(type, typename, _params.ShowReadOnlyProperties); foreach (var kv in d) { var n = kv.Key; var v = kv.Value; string name = n;//.ToLower(); if (name == "$map") { ProcessMap(o, props, (Dictionary)d[name]); continue; } myPropInfo pi; if (props.TryGetValue(name, out pi) == false) if (props.TryGetValue(name.ToLowerInvariant(), out pi) == false) continue; if (pi.CanWrite) { if (v != null) { object oset = null; switch (pi.Type) { case myPropInfoType.Int: oset = (int)Helper.AutoConv(v, _params); break; case myPropInfoType.Long: oset = Helper.AutoConv(v, _params); break; case myPropInfoType.String: oset = v.ToString(); break; case myPropInfoType.Bool: oset = Helper.BoolConv(v); break; case myPropInfoType.DateTime: oset = Helper.CreateDateTime((string)v, _params.UseUTCDateTime); break; case myPropInfoType.Enum: oset = Helper.CreateEnum(pi.pt, v); break; case myPropInfoType.Guid: oset = Helper.CreateGuid((string)v); break; case myPropInfoType.Array: if (!pi.IsValueType) oset = CreateArray((List)v, pi.pt, pi.bt, globaltypes); // what about 'else'? break; case myPropInfoType.ByteArray: oset = Convert.FromBase64String((string)v); break; #if !SILVERLIGHT case myPropInfoType.DataSet: oset = CreateDataset((Dictionary)v, globaltypes); break; case myPropInfoType.DataTable: oset = CreateDataTable((Dictionary)v, globaltypes); break; case myPropInfoType.Hashtable: // same case as Dictionary #endif case myPropInfoType.Dictionary: oset = CreateDictionary((List)v, pi.pt, pi.GenericTypes, globaltypes); break; case myPropInfoType.StringKeyDictionary: oset = CreateStringKeyDictionary((Dictionary)v, pi.pt, pi.GenericTypes, globaltypes); break; case myPropInfoType.NameValue: oset = Helper.CreateNV((Dictionary)v); break; case myPropInfoType.StringDictionary: oset = Helper.CreateSD((Dictionary)v); break; case myPropInfoType.Custom: oset = Reflection.Instance.CreateCustom((string)v, pi.pt); break; default: { if (pi.IsGenericType && pi.IsValueType == false && v is List) oset = CreateGenericList((List)v, pi.pt, pi.bt, globaltypes); else if ((pi.IsClass || pi.IsStruct || pi.IsInterface) && v is Dictionary) oset = ParseDictionary((Dictionary)v, globaltypes, pi.pt, null);// pi.getter(o)); else if (v is List) oset = CreateArray((List)v, pi.pt, typeof(object), globaltypes); else if (pi.IsValueType) oset = ChangeType(v, pi.changeType); else oset = v; } break; } o = pi.setter(o, oset); } } } return o; } private static void ProcessMap(object obj, Dictionary props, Dictionary dic) { foreach (KeyValuePair kv in dic) { myPropInfo p = props[kv.Key]; object o = p.getter(obj); Type t = Type.GetType((string)kv.Value); if (t == typeof(Guid)) p.setter(obj, Helper.CreateGuid((string)o)); } } private object CreateArray(List data, Type pt, Type bt, Dictionary globalTypes) { if (bt == null) bt = typeof(object); Array col = Array.CreateInstance(bt, data.Count); var arraytype = bt.GetElementType(); // create an array of objects for (int i = 0; i < data.Count; i++) { object ob = data[i]; if (ob == null) { continue; } if (ob is IDictionary) col.SetValue(ParseDictionary((Dictionary)ob, globalTypes, bt, null), i); else if (ob is ICollection) col.SetValue(CreateArray((List)ob, bt, arraytype, globalTypes), i); else col.SetValue(ChangeType(ob, bt), i); } return col; } private object CreateGenericList(List data, Type pt, Type bt, Dictionary globalTypes) { if (pt != typeof(object)) { IList col = (IList)Reflection.Instance.FastCreateList(pt, data.Count); var it = Reflection.Instance.GetGenericArguments(pt)[0];// pt.GetGenericArguments()[0]; // create an array of objects foreach (object ob in data) { if (ob is IDictionary) col.Add(ParseDictionary((Dictionary)ob, globalTypes, it, null)); else if (ob is List) { if (bt.IsGenericType) col.Add((List)ob);//).ToArray()); else col.Add(((List)ob).ToArray()); } else col.Add(ChangeType(ob, it)); } return col; } return data; } private object CreateStringKeyDictionary(Dictionary reader, Type pt, Type[] types, Dictionary globalTypes) { var col = (IDictionary)Reflection.Instance.FastCreateInstance(pt); Type arraytype = null; Type t2 = null; if (types != null) t2 = types[1]; Type generictype = null; var ga = Reflection.Instance.GetGenericArguments(t2);// t2.GetGenericArguments(); if (ga.Length > 0) generictype = ga[0]; arraytype = t2.GetElementType(); foreach (KeyValuePair values in reader) { var key = values.Key; object val = null; if (values.Value is Dictionary) val = ParseDictionary((Dictionary)values.Value, globalTypes, t2, null); else if (types != null && t2.IsArray) { if (values.Value is Array) val = values.Value; else val = CreateArray((List)values.Value, t2, arraytype, globalTypes); } else if (values.Value is IList) val = CreateGenericList((List)values.Value, t2, generictype, globalTypes); else val = ChangeType(values.Value, t2); col.Add(key, val); } return col; } private object CreateDictionary(List reader, Type pt, Type[] types, Dictionary globalTypes) { IDictionary col = (IDictionary)Reflection.Instance.FastCreateInstance(pt); Type t1 = null; Type t2 = null; Type generictype = null; if (types != null) { t1 = types[0]; t2 = types[1]; } Type arraytype = t2; if (t2 != null) { var ga = Reflection.Instance.GetGenericArguments(t2);// t2.GetGenericArguments(); if (ga.Length > 0) generictype = ga[0]; arraytype = t2.GetElementType(); } bool root = typeof(IDictionary).IsAssignableFrom(t2); foreach (Dictionary values in reader) { object key = values["k"]; object val = values["v"]; if (key is Dictionary) key = ParseDictionary((Dictionary)key, globalTypes, t1, null); else key = ChangeType(key, t1); if (root) val = RootDictionary(val, t2); else if (val is Dictionary) val = ParseDictionary((Dictionary)val, globalTypes, t2, null); else if (types != null && t2.IsArray) val = CreateArray((List)val, t2, arraytype, globalTypes); else if (val is IList) val = CreateGenericList((List)val, t2, generictype, globalTypes); else val = ChangeType(val, t2); col.Add(key, val); } return col; } #if !SILVERLIGHT private DataSet CreateDataset(Dictionary reader, Dictionary globalTypes) { DataSet ds = new DataSet(); ds.EnforceConstraints = false; ds.BeginInit(); // read dataset schema here var schema = reader["$schema"]; if (schema is string) { TextReader tr = new StringReader((string)schema); ds.ReadXmlSchema(tr); } else { DatasetSchema ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null); ds.DataSetName = ms.Name; for (int i = 0; i < ms.Info.Count; i += 3) { if (ds.Tables.Contains(ms.Info[i]) == false) ds.Tables.Add(ms.Info[i]); ds.Tables[ms.Info[i]].Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2])); } } foreach (KeyValuePair pair in reader) { if (pair.Key == "$type" || pair.Key == "$schema") continue; List rows = (List)pair.Value; if (rows == null) continue; DataTable dt = ds.Tables[pair.Key]; ReadDataTable(rows, dt); } ds.EndInit(); return ds; } private void ReadDataTable(List rows, DataTable dt) { dt.BeginInit(); dt.BeginLoadData(); List guidcols = new List(); List datecol = new List(); List bytearraycol = new List(); foreach (DataColumn c in dt.Columns) { if (c.DataType == typeof(Guid) || c.DataType == typeof(Guid?)) guidcols.Add(c.Ordinal); if (_params.UseUTCDateTime && (c.DataType == typeof(DateTime) || c.DataType == typeof(DateTime?))) datecol.Add(c.Ordinal); if (c.DataType == typeof(byte[])) bytearraycol.Add(c.Ordinal); } foreach (List row in rows) { object[] v = new object[row.Count]; row.CopyTo(v, 0); foreach (int i in guidcols) { string s = (string)v[i]; if (s != null && s.Length < 36) v[i] = new Guid(Convert.FromBase64String(s)); } foreach (int i in bytearraycol) { string s = (string)v[i]; if (s != null) v[i] = Convert.FromBase64String(s); } if (_params.UseUTCDateTime) { foreach (int i in datecol) { string s = (string)v[i]; if (s != null) v[i] = Helper.CreateDateTime(s, _params.UseUTCDateTime); } } dt.Rows.Add(v); } dt.EndLoadData(); dt.EndInit(); } DataTable CreateDataTable(Dictionary reader, Dictionary globalTypes) { var dt = new DataTable(); // read dataset schema here var schema = reader["$schema"]; if (schema is string) { TextReader tr = new StringReader((string)schema); dt.ReadXmlSchema(tr); } else { var ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null); dt.TableName = ms.Info[0]; for (int i = 0; i < ms.Info.Count; i += 3) { dt.Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2])); } } foreach (var pair in reader) { if (pair.Key == "$type" || pair.Key == "$schema") continue; var rows = (List)pair.Value; if (rows == null) continue; if (!dt.TableName.Equals(pair.Key, StringComparison.InvariantCultureIgnoreCase)) continue; ReadDataTable(rows, dt); } return dt; } #endif #endregion } } ================================================ FILE: RaptorDB.Common/fastJSON/JsonParser.cs ================================================ using System; using System.Collections.Generic; using System.Globalization; using System.Text; namespace fastJSON { /// /// This class encodes and decodes JSON strings. /// Spec. details, see http://www.json.org/ /// internal sealed class JsonParser { enum Token { None = -1, // Used to denote no Lookahead available Curly_Open, Curly_Close, Squared_Open, Squared_Close, Colon, Comma, String, Number, True, False, Null//, //Key } readonly char[] json; readonly StringBuilder s = new StringBuilder(); // used for inner string parsing " \"\r\n\u1234\'\t " Token lookAheadToken = Token.None; int index; bool allownonquotedkey = false; int _len = 0; internal JsonParser(string json, bool AllowNonQuotedKeys) { this.allownonquotedkey = AllowNonQuotedKeys; this.json = json.ToCharArray(); _len = json.Length; } public unsafe object Decode() { fixed(char* p = json) return ParseValue(p, false); } private unsafe Dictionary ParseObject(char* p) { Dictionary table = new Dictionary(10); ConsumeToken(); // { while (true) { switch (LookAhead(p)) { case Token.Comma: ConsumeToken(); break; case Token.Curly_Close: ConsumeToken(); return table; default: { // name string name = ParseString(p, false); var n = NextToken(p); // : if (n != Token.Colon) { throw new Exception("Expected colon at index " + index); } // value object value = ParseValue(p, true); //table.Add(name, value); table[name] = value; } break; } } } private unsafe List ParseArray(char* p) { List array = new List(10); ConsumeToken(); // [ while (true) { switch (LookAhead(p)) { case Token.Comma: ConsumeToken(); break; case Token.Squared_Close: ConsumeToken(); return array; default: array.Add(ParseValue(p, false)); break; } } } private unsafe object ParseValue(char* p, bool val) { switch (LookAhead(p)) { case Token.Number: return ParseNumber(p); case Token.String: return ParseString(p, val); case Token.Curly_Open: return ParseObject(p); case Token.Squared_Open: return ParseArray(p); case Token.True: ConsumeToken(); return true; case Token.False: ConsumeToken(); return false; case Token.Null: ConsumeToken(); return null; } throw new Exception("Unrecognized token at index" + index); } private unsafe string ParseString(char* p, bool val) { ConsumeToken(); // " if (s.Length > 0) s.Length = 0; //s.Clear(); bool instr = val; int runIndex = -1; int l = _len; //fixed (char* p = json) //char[] p = json; { while (index < l) { var c = p[index++]; if (c == '"') instr = true; if (c == '"' || (allownonquotedkey && instr == false && (c == ':' || c == ' ' || c == '\t'))) { int len = 1; if (allownonquotedkey && c != '"' && instr == false) { index--; len = 0; } if (runIndex != -1) { if (s.Length == 0) return UnsafeSubstring(p, runIndex, index - runIndex - len); s.Append(json, runIndex, index - runIndex - 1); } return s.ToString(); } if (c != '\\') { if (runIndex == -1) runIndex = index - 1; continue; } if (index == l) break; if (runIndex != -1) { s.Append(json, runIndex, index - runIndex - 1); runIndex = -1; } switch (p[index++]) { case '"': s.Append('"'); break; case '\\': s.Append('\\'); break; case '/': s.Append('/'); break; case 'b': s.Append('\b'); break; case 'f': s.Append('\f'); break; case 'n': s.Append('\n'); break; case 'r': s.Append('\r'); break; case 't': s.Append('\t'); break; case 'u': { int remainingLength = l - index; if (remainingLength < 4) break; // parse the 32 bit hex into an integer codepoint uint codePoint = ParseUnicode(p[index], p[index + 1], p[index + 2], p[index + 3]); s.Append((char)codePoint); // skip 4 chars index += 4; } break; } } } throw new Exception("Unexpectedly reached end of string"); } private uint ParseSingleChar(char c1, uint multipliyer) { uint p1 = 0; if (c1 >= '0' && c1 <= '9') p1 = (uint)(c1 - '0') * multipliyer; else if (c1 >= 'A' && c1 <= 'F') p1 = (uint)((c1 - 'A') + 10) * multipliyer; else if (c1 >= 'a' && c1 <= 'f') p1 = (uint)((c1 - 'a') + 10) * multipliyer; return p1; } private uint ParseUnicode(char c1, char c2, char c3, char c4) { uint p1 = ParseSingleChar(c1, 0x1000); uint p2 = ParseSingleChar(c2, 0x100); uint p3 = ParseSingleChar(c3, 0x10); uint p4 = ParseSingleChar(c4, 1); return p1 + p2 + p3 + p4; } private unsafe object ParseNumber(char* p) { ConsumeToken(); // Need to start back one place because the first digit is also a token and would have been consumed var startIndex = index - 1; bool dec = false; bool dob = false; do { if (index == _len) break; var c = json[index]; if ((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E') { if (/*c == '.' ||*/ c == 'e' || c == 'E') dob = true; if (c == '.') dec = true; if (++index == _len) break;//throw new Exception("Unexpected end of string whilst parsing number"); continue; } break; } while (true); if (dob) { string s = UnsafeSubstring(p, startIndex, index - startIndex);// json.Substring(startIndex, index - startIndex); return double.Parse(s, NumberFormatInfo.InvariantInfo); } if (dec == false && index - startIndex < 20 )// && json[startIndex] != '-') return Helper.CreateLong(json, startIndex, index - startIndex); else { string s = UnsafeSubstring(p, startIndex, index - startIndex);//json.Substring(startIndex, index - startIndex); return decimal.Parse(s, NumberFormatInfo.InvariantInfo); } } private unsafe Token LookAhead(char* p) { if (lookAheadToken != Token.None) return lookAheadToken; return lookAheadToken = NextTokenCore(p); } private void ConsumeToken() { lookAheadToken = Token.None; } private unsafe Token NextToken(char* p) { var result = lookAheadToken != Token.None ? lookAheadToken : NextTokenCore(p); lookAheadToken = Token.None; return result; } private unsafe Token NextTokenCore(char* p) { char c; int len = _len; // Skip past whitespace do { //fixed (char* p = json) { c = p[index]; if (c == '/' && p[index + 1] == '/') // c++ style single line comments { index++; index++; do { c = p[index]; if (c == '\r' || c == '\n') break; // read till end of line } while (++index < len); } if (c > ' ') break; if (c != ' ' && c != '\t' && c != '\n' && c != '\r') break; } } while (++index < len); if (index == len) { throw new Exception("Reached end of string unexpectedly"); } //fixed (char* p = json) { c = p[index]; index++; switch (c) { case '{': return Token.Curly_Open; case '}': return Token.Curly_Close; case '[': return Token.Squared_Open; case ']': return Token.Squared_Close; case ',': return Token.Comma; case '"': return Token.String; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': case '-': case '+': case '.': return Token.Number; case ':': return Token.Colon; case 'f': if (len - index >= 4 && p[index + 0] == 'a' && p[index + 1] == 'l' && p[index + 2] == 's' && p[index + 3] == 'e') { index += 4; return Token.False; } break; case 't': if (len - index >= 3 && p[index + 0] == 'r' && p[index + 1] == 'u' && p[index + 2] == 'e') { index += 3; return Token.True; } break; case 'n': if (len - index >= 3 && p[index + 0] == 'u' && p[index + 1] == 'l' && p[index + 2] == 'l') { index += 3; return Token.Null; } break; } } if (allownonquotedkey) { index--; return Token.String; } else throw new Exception("Could not find token at index " + --index); } private static unsafe string UnsafeSubstring(//char[] source, char* p, int startIndex, int length) { //fixed (char* c = source) return new string(p, startIndex, length); } } } ================================================ FILE: RaptorDB.Common/fastJSON/JsonSerializer.cs ================================================ using System; using System.Collections; using System.Collections.Generic; #if !SILVERLIGHT using System.Data; #endif using System.Globalization; using System.IO; using System.Text; using System.Collections.Specialized; using RaptorDB.Common; namespace fastJSON { internal sealed class JSONSerializer { private StringBuilder _output = new StringBuilder(); //private StringBuilder _before = new StringBuilder(); private int _before; private int _MAX_DEPTH = 20; int _current_depth = 0; private Dictionary _globalTypes = new Dictionary(); private Dictionary _cirobj; private JSONParameters _params; private bool _useEscapedUnicode = false; internal JSONSerializer(JSONParameters param) { if (param.OverrideObjectHashCodeChecking) _cirobj = new Dictionary(10, ReferenceEqualityComparer.Default); else _cirobj = new Dictionary(); _params = param; _useEscapedUnicode = _params.UseEscapedUnicode; _MAX_DEPTH = _params.SerializerMaxDepth; } internal string ConvertToJSON(object obj) { WriteValue(obj); if (_params.UsingGlobalTypes && _globalTypes != null && _globalTypes.Count > 0) { var sb = new StringBuilder(); sb.Append("\"$types\":{"); var pendingSeparator = false; foreach (var kv in _globalTypes) { if (pendingSeparator) sb.Append(','); pendingSeparator = true; sb.Append('\"'); sb.Append(kv.Key); sb.Append("\":\""); sb.Append(kv.Value); sb.Append('\"'); } sb.Append("},"); _output.Insert(_before, sb.ToString()); } return _output.ToString(); } private void WriteValue(object obj) { if (obj == null || obj is DBNull) _output.Append("null"); else if (obj is string || obj is char) WriteString(obj.ToString()); else if (obj is Guid) WriteGuid((Guid)obj); else if (obj is bool) _output.Append(((bool)obj) ? "true" : "false"); // conform to standard else if ( obj is int || obj is long || obj is decimal || obj is byte || obj is short || obj is sbyte || obj is ushort || obj is uint || obj is ulong ) _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo)); else if (obj is double || obj is Double) { double d = (double)obj; if (double.IsNaN(d)) _output.Append("\"NaN\""); else if (double.IsInfinity(d)) { _output.Append("\""); _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo)); _output.Append("\""); } else _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo)); } else if (obj is float || obj is Single) { float d = (float)obj; if (float.IsNaN(d)) _output.Append("\"NaN\""); else if (float.IsInfinity(d)) { _output.Append("\""); _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo)); _output.Append("\""); } else _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo)); } else if (obj is DateTime) WriteDateTime((DateTime)obj); else if (obj is DateTimeOffset) WriteDateTimeOffset((DateTimeOffset)obj); else if (obj is TimeSpan) _output.Append(((TimeSpan)obj).Ticks); #if NET4 else if (_params.KVStyleStringDictionary == false && obj is IEnumerable>) WriteStringDictionary((IEnumerable>)obj); #endif else if (_params.KVStyleStringDictionary == false && obj is IDictionary && obj.GetType().IsGenericType && Reflection.Instance.GetGenericArguments(obj.GetType())[0] == typeof(string)) WriteStringDictionary((IDictionary)obj); else if (obj is IDictionary) WriteDictionary((IDictionary)obj); #if !SILVERLIGHT else if (obj is DataSet) WriteDataset((DataSet)obj); else if (obj is DataTable) this.WriteDataTable((DataTable)obj); #endif else if (obj is byte[]) WriteBytes((byte[])obj); else if (obj is StringDictionary) WriteSD((StringDictionary)obj); else if (obj is NameValueCollection) WriteNV((NameValueCollection)obj); else if (obj is IEnumerable) WriteArray((IEnumerable)obj); else if (obj is Enum) WriteEnum((Enum)obj); else if (Reflection.Instance.IsTypeRegistered(obj.GetType())) WriteCustom(obj); else WriteObject(obj); } private void WriteDateTimeOffset(DateTimeOffset d) { DateTime dt = _params.UseUTCDateTime ? d.UtcDateTime : d.DateTime; write_date_value(dt); var ticks = dt.Ticks % TimeSpan.TicksPerSecond; _output.Append('.'); _output.Append(ticks.ToString("0000000", NumberFormatInfo.InvariantInfo)); if (_params.UseUTCDateTime) _output.Append('Z'); else { if (d.Offset.Hours > 0) _output.Append("+"); else _output.Append("-"); _output.Append(d.Offset.Hours.ToString("00", NumberFormatInfo.InvariantInfo)); _output.Append(":"); _output.Append(d.Offset.Minutes.ToString("00", NumberFormatInfo.InvariantInfo)); } _output.Append('\"'); } private void WriteNV(NameValueCollection nameValueCollection) { _output.Append('{'); bool pendingSeparator = false; foreach (string key in nameValueCollection) { if (_params.SerializeNullValues == false && (nameValueCollection[key] == null)) { } else { if (pendingSeparator) _output.Append(','); if (_params.SerializeToLowerCaseNames) WritePair(key.ToLowerInvariant(), nameValueCollection[key]); else WritePair(key, nameValueCollection[key]); pendingSeparator = true; } } _output.Append('}'); } private void WriteSD(StringDictionary stringDictionary) { _output.Append('{'); bool pendingSeparator = false; foreach (DictionaryEntry entry in stringDictionary) { if (_params.SerializeNullValues == false && (entry.Value == null)) { } else { if (pendingSeparator) _output.Append(','); string k = (string)entry.Key; if (_params.SerializeToLowerCaseNames) WritePair(k.ToLowerInvariant(), entry.Value); else WritePair(k, entry.Value); pendingSeparator = true; } } _output.Append('}'); } private void WriteCustom(object obj) { Reflection.Serialize s; Reflection.Instance._customSerializer.TryGetValue(obj.GetType(), out s); WriteStringFast(s(obj)); } private void WriteEnum(Enum e) { // FEATURE : optimize enum write if (_params.UseValuesOfEnums) WriteValue(Convert.ToInt32(e)); else WriteStringFast(e.ToString()); } private void WriteGuid(Guid g) { if (_params.UseFastGuid == false) WriteStringFast(g.ToString()); else WriteBytes(g.ToByteArray()); } private void WriteBytes(byte[] bytes) { #if !SILVERLIGHT WriteStringFast(Convert.ToBase64String(bytes, 0, bytes.Length, Base64FormattingOptions.None)); #else WriteStringFast(Convert.ToBase64String(bytes, 0, bytes.Length)); #endif } private void WriteDateTime(DateTime dateTime) { // datetime format standard : yyyy-MM-dd HH:mm:ss DateTime dt = dateTime; if (_params.UseUTCDateTime) dt = dateTime.ToUniversalTime(); write_date_value(dt); if (_params.DateTimeMilliseconds) { _output.Append('.'); _output.Append(dt.Millisecond.ToString("000", NumberFormatInfo.InvariantInfo)); } if (_params.UseUTCDateTime) _output.Append('Z'); _output.Append('\"'); } private void write_date_value(DateTime dt) { _output.Append('\"'); _output.Append(dt.Year.ToString("0000", NumberFormatInfo.InvariantInfo)); _output.Append('-'); _output.Append(dt.Month.ToString("00", NumberFormatInfo.InvariantInfo)); _output.Append('-'); _output.Append(dt.Day.ToString("00", NumberFormatInfo.InvariantInfo)); _output.Append('T'); // strict ISO date compliance _output.Append(dt.Hour.ToString("00", NumberFormatInfo.InvariantInfo)); _output.Append(':'); _output.Append(dt.Minute.ToString("00", NumberFormatInfo.InvariantInfo)); _output.Append(':'); _output.Append(dt.Second.ToString("00", NumberFormatInfo.InvariantInfo)); } #if !SILVERLIGHT private DatasetSchema GetSchema(DataTable ds) { if (ds == null) return null; DatasetSchema m = new DatasetSchema(); m.Info = new List(); m.Name = ds.TableName; foreach (DataColumn c in ds.Columns) { m.Info.Add(ds.TableName); m.Info.Add(c.ColumnName); m.Info.Add(c.DataType.ToString()); } // FEATURE : serialize relations and constraints here return m; } private DatasetSchema GetSchema(DataSet ds) { if (ds == null) return null; DatasetSchema m = new DatasetSchema(); m.Info = new List(); m.Name = ds.DataSetName; foreach (DataTable t in ds.Tables) { foreach (DataColumn c in t.Columns) { m.Info.Add(t.TableName); m.Info.Add(c.ColumnName); m.Info.Add(c.DataType.ToString()); } } // FEATURE : serialize relations and constraints here return m; } private string GetXmlSchema(DataTable dt) { using (var writer = new StringWriter()) { dt.WriteXmlSchema(writer); return dt.ToString(); } } private void WriteDataset(DataSet ds) { _output.Append('{'); if (_params.UseExtensions) { WritePair("$schema", _params.UseOptimizedDatasetSchema ? (object)GetSchema(ds) : ds.GetXmlSchema()); _output.Append(','); } bool tablesep = false; foreach (DataTable table in ds.Tables) { if (tablesep) _output.Append(','); tablesep = true; WriteDataTableData(table); } // end dataset _output.Append('}'); } private void WriteDataTableData(DataTable table) { _output.Append('\"'); _output.Append(table.TableName); _output.Append("\":["); DataColumnCollection cols = table.Columns; bool rowseparator = false; foreach (DataRow row in table.Rows) { if (rowseparator) _output.Append(','); rowseparator = true; _output.Append('['); bool pendingSeperator = false; foreach (DataColumn column in cols) { if (pendingSeperator) _output.Append(','); WriteValue(row[column]); pendingSeperator = true; } _output.Append(']'); } _output.Append(']'); } void WriteDataTable(DataTable dt) { this._output.Append('{'); if (_params.UseExtensions) { this.WritePair("$schema", _params.UseOptimizedDatasetSchema ? (object)this.GetSchema(dt) : this.GetXmlSchema(dt)); this._output.Append(','); } WriteDataTableData(dt); // end datatable this._output.Append('}'); } #endif bool _TypesWritten = false; private void WriteObject(object obj) { int i = 0; if (_cirobj.TryGetValue(obj, out i) == false) _cirobj.Add(obj, _cirobj.Count + 1); else { if (_current_depth > 0 && _params.InlineCircularReferences == false) { //_circular = true; _output.Append("{\"$i\":"); _output.Append(i.ToString()); _output.Append("}"); return; } } if (_params.UsingGlobalTypes == false) _output.Append('{'); else { if (_TypesWritten == false) { _output.Append('{'); _before = _output.Length; //_output = new StringBuilder(); } else _output.Append('{'); } _TypesWritten = true; _current_depth++; if (_current_depth > _MAX_DEPTH) throw new Exception("Serializer encountered maximum depth of " + _MAX_DEPTH); Dictionary map = new Dictionary(); Type t = obj.GetType(); bool append = false; if (_params.UseExtensions) { if (_params.UsingGlobalTypes == false) WritePairFast("$type", Reflection.Instance.GetTypeAssemblyName(t)); else { int dt = 0; string ct = Reflection.Instance.GetTypeAssemblyName(t); if (_globalTypes.TryGetValue(ct, out dt) == false) { dt = _globalTypes.Count + 1; _globalTypes.Add(ct, dt); } WritePairFast("$type", dt.ToString()); } append = true; } Getters[] g = Reflection.Instance.GetGetters(t, /*_params.ShowReadOnlyProperties,*/ _params.IgnoreAttributes); int c = g.Length; for (int ii = 0; ii < c; ii++) { var p = g[ii]; if (_params.ShowReadOnlyProperties == false && p.ReadOnly) continue; object o = p.Getter(obj); if (_params.SerializeNullValues == false && (o == null || o is DBNull)) { //append = false; } else { if (append) _output.Append(','); if (p.memberName != null) WritePair(p.memberName, o); else if (_params.SerializeToLowerCaseNames) WritePair(p.lcName, o); else WritePair(p.Name, o); if (o != null && _params.UseExtensions) { Type tt = o.GetType(); if (tt == typeof(System.Object)) map.Add(p.Name, tt.ToString()); } append = true; } } if (map.Count > 0 && _params.UseExtensions) { _output.Append(",\"$map\":"); WriteStringDictionary(map); } _output.Append('}'); _current_depth--; } private void WritePairFast(string name, string value) { WriteStringFast(name); _output.Append(':'); WriteStringFast(value); } private void WritePair(string name, object value) { WriteString(name); _output.Append(':'); WriteValue(value); } private void WriteArray(IEnumerable array) { _output.Append('['); bool pendingSeperator = false; foreach (object obj in array) { if (pendingSeperator) _output.Append(','); WriteValue(obj); pendingSeperator = true; } _output.Append(']'); } private void WriteStringDictionary(IDictionary dic) { _output.Append('{'); bool pendingSeparator = false; foreach (DictionaryEntry entry in dic) { if (_params.SerializeNullValues == false && (entry.Value == null)) { } else { if (pendingSeparator) _output.Append(','); string k = (string)entry.Key; if (_params.SerializeToLowerCaseNames) WritePair(k.ToLowerInvariant(), entry.Value); else WritePair(k, entry.Value); pendingSeparator = true; } } _output.Append('}'); } private void WriteStringDictionary(IEnumerable> dic) { _output.Append('{'); bool pendingSeparator = false; foreach (KeyValuePair entry in dic) { if (_params.SerializeNullValues == false && (entry.Value == null)) { } else { if (pendingSeparator) _output.Append(','); string k = entry.Key; if (_params.SerializeToLowerCaseNames) WritePair(k.ToLowerInvariant(), entry.Value); else WritePair(k, entry.Value); pendingSeparator = true; } } _output.Append('}'); } private void WriteDictionary(IDictionary dic) { _output.Append('['); bool pendingSeparator = false; foreach (DictionaryEntry entry in dic) { if (pendingSeparator) _output.Append(','); _output.Append('{'); WritePair("k", entry.Key); _output.Append(","); WritePair("v", entry.Value); _output.Append('}'); pendingSeparator = true; } _output.Append(']'); } private void WriteStringFast(string s) { _output.Append('\"'); _output.Append(s); _output.Append('\"'); } private void WriteString(string s) { _output.Append('\"'); int runIndex = -1; int l = s.Length; for (var index = 0; index < l; ++index) { var c = s[index]; if (_useEscapedUnicode) { if (c >= ' ' && c < 128 && c != '\"' && c != '\\') { if (runIndex == -1) runIndex = index; continue; } } else { if (c != '\t' && c != '\n' && c != '\r' && c != '\"' && c != '\\' && c!='\0')// && c != ':' && c!=',') { if (runIndex == -1) runIndex = index; continue; } } if (runIndex != -1) { _output.Append(s, runIndex, index - runIndex); runIndex = -1; } switch (c) { case '\t': _output.Append("\\t"); break; case '\r': _output.Append("\\r"); break; case '\n': _output.Append("\\n"); break; case '"': case '\\': _output.Append('\\'); _output.Append(c); break; case '\0': _output.Append("\\u0000"); break; default: if (_useEscapedUnicode) { _output.Append("\\u"); _output.Append(((int)c).ToString("X4", NumberFormatInfo.InvariantInfo)); } else _output.Append(c); break; } } if (runIndex != -1) _output.Append(s, runIndex, s.Length - runIndex); _output.Append('\"'); } } } ================================================ FILE: RaptorDB.Common/fastJSON/Reflection.cs ================================================ using System; using System.Collections.Generic; using System.Reflection.Emit; using System.Reflection; using System.Collections; using System.Text; using System.Runtime.Serialization; #if NET4 using System.Linq; using RaptorDB.Common; #endif #if !SILVERLIGHT using System.Data; #endif using System.Collections.Specialized; namespace fastJSON { public struct Getters { public string Name; public string lcName; public string memberName; public Reflection.GenericGetter Getter; public bool ReadOnly; } public enum myPropInfoType { Int, Long, String, Bool, DateTime, Enum, Guid, Array, ByteArray, Dictionary, StringKeyDictionary, NameValue, StringDictionary, #if !SILVERLIGHT Hashtable, DataSet, DataTable, #endif Custom, Unknown, } public class myPropInfo { public Type pt; public Type bt; public Type changeType; public Reflection.GenericSetter setter; public Reflection.GenericGetter getter; public Type[] GenericTypes; public string Name; #if NET4 public string memberName; #endif public myPropInfoType Type; public bool CanWrite; public bool IsClass; public bool IsValueType; public bool IsGenericType; public bool IsStruct; public bool IsInterface; } public sealed class Reflection { // Singleton pattern 4 from : http://csharpindepth.com/articles/general/singleton.aspx private static readonly Reflection instance = new Reflection(); // Explicit static constructor to tell C# compiler // not to mark type as beforefieldinit static Reflection() { } private Reflection() { } public static Reflection Instance { get { return instance; } } public static bool RDBMode = false; public delegate string Serialize(object data); public delegate object Deserialize(string data); public delegate object GenericSetter(object target, object value); public delegate object GenericGetter(object obj); private delegate object CreateObject(); private delegate object CreateList(int capacity); private SafeDictionary _tyname = new SafeDictionary(10); private SafeDictionary _typecache = new SafeDictionary(10); private SafeDictionary _constrcache = new SafeDictionary(10); private SafeDictionary _conlistcache = new SafeDictionary(10); private SafeDictionary _getterscache = new SafeDictionary(10); private SafeDictionary> _propertycache = new SafeDictionary>(10); private SafeDictionary _genericTypes = new SafeDictionary(10); private SafeDictionary _genericTypeDef = new SafeDictionary(10); private static SafeDictionary _opCodes; private static List _blacklistTypes = new List() { "system.configuration.install.assemblyinstaller", "system.activities.presentation.workflowdesigner", "system.windows.resourcedictionary", "system.windows.data.objectdataprovider", "system.windows.forms.bindingsource", "microsoft.exchange.management.systemmanager.winforms.exchangesettingsprovider" }; private static bool TryGetOpCode(short code, out OpCode opCode) { if (_opCodes != null) return _opCodes.TryGetValue(code, out opCode); var dict = new SafeDictionary(); foreach (var fi in typeof(OpCodes).GetFields(BindingFlags.Public | BindingFlags.Static)) { if (!typeof(OpCode).IsAssignableFrom(fi.FieldType)) continue; var innerOpCode = (OpCode)fi.GetValue(null); if (innerOpCode.OpCodeType != OpCodeType.Nternal) dict.Add(innerOpCode.Value, innerOpCode); } _opCodes = dict; return _opCodes.TryGetValue(code, out opCode); } #region bjson custom types //internal UnicodeEncoding unicode = new UnicodeEncoding(); private static UTF8Encoding utf8 = new UTF8Encoding(); // TODO : optimize utf8 public static byte[] UTF8GetBytes(string str) { return utf8.GetBytes(str); } public static string UTF8GetString(byte[] bytes, int offset, int len) { return utf8.GetString(bytes, offset, len); } public unsafe static byte[] UnicodeGetBytes(string str) { int len = str.Length * 2; byte[] b = new byte[len]; fixed (void* ptr = str) { System.Runtime.InteropServices.Marshal.Copy(new IntPtr(ptr), b, 0, len); } return b; } public static string UnicodeGetString(byte[] b) { return UnicodeGetString(b, 0, b.Length); } public unsafe static string UnicodeGetString(byte[] bytes, int offset, int buflen) { string str = ""; fixed (byte* bptr = bytes) { char* cptr = (char*)(bptr + offset); str = new string(cptr, 0, buflen / 2); } return str; } #endregion #region json custom types // JSON custom internal SafeDictionary _customSerializer = new SafeDictionary(); internal SafeDictionary _customDeserializer = new SafeDictionary(); internal object CreateCustom(string v, Type type) { Deserialize d; _customDeserializer.TryGetValue(type, out d); return d(v); } internal void RegisterCustomType(Type type, Serialize serializer, Deserialize deserializer) { if (type != null && serializer != null && deserializer != null) { _customSerializer.Add(type, serializer); _customDeserializer.Add(type, deserializer); // reset property cache Instance.ResetPropertyCache(); } } internal bool IsTypeRegistered(Type t) { if (_customSerializer.Count() == 0) return false; Serialize s; return _customSerializer.TryGetValue(t, out s); } #endregion public Type GetGenericTypeDefinition(Type t) { Type tt = null; if (_genericTypeDef.TryGetValue(t, out tt)) return tt; else { tt = t.GetGenericTypeDefinition(); _genericTypeDef.Add(t, tt); return tt; } } public Type[] GetGenericArguments(Type t) { Type[] tt = null; if (_genericTypes.TryGetValue(t, out tt)) return tt; else { tt = t.GetGenericArguments(); _genericTypes.Add(t, tt); return tt; } } public Dictionary Getproperties(Type type, string typename, bool ShowReadOnlyProperties) { Dictionary sd = null; if (_propertycache.TryGetValue(typename, out sd)) { return sd; } else { sd = new Dictionary(10); var bf = BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static; PropertyInfo[] pr = type.GetProperties(bf); foreach (PropertyInfo p in pr) { if (p.GetIndexParameters().Length > 0)// Property is an indexer continue; myPropInfo d = CreateMyProp(p.PropertyType, p.Name); d.setter = Reflection.CreateSetMethod(type, p, ShowReadOnlyProperties); if (d.setter != null) d.CanWrite = true; d.getter = Reflection.CreateGetMethod(type, p); #if NET4 var att = p.GetCustomAttributes(true); foreach (var at in att) { if (at is DataMemberAttribute) { var dm = (DataMemberAttribute)at; if (dm.Name != "") d.memberName = dm.Name; } } if (d.memberName != null) sd.Add(d.memberName, d); else #endif sd.Add(p.Name.ToLowerInvariant(), d); } FieldInfo[] fi = type.GetFields(bf); foreach (FieldInfo f in fi) { myPropInfo d = CreateMyProp(f.FieldType, f.Name); if (f.IsLiteral == false) { if (f.IsInitOnly == false) d.setter = Reflection.CreateSetField(type, f); if (d.setter != null) d.CanWrite = true; d.getter = Reflection.CreateGetField(type, f); #if NET4 var att = f.GetCustomAttributes(true); foreach (var at in att) { if (at is DataMemberAttribute) { var dm = (DataMemberAttribute)at; if (dm.Name != "") d.memberName = dm.Name; } } if (d.memberName != null) sd.Add(d.memberName, d); else #endif sd.Add(f.Name.ToLowerInvariant(), d); } } _propertycache.Add(typename, sd); return sd; } } private myPropInfo CreateMyProp(Type t, string name) { myPropInfo d = new myPropInfo(); myPropInfoType d_type = myPropInfoType.Unknown; if (t == typeof(int) || t == typeof(int?)) d_type = myPropInfoType.Int; else if (t == typeof(long) || t == typeof(long?)) d_type = myPropInfoType.Long; else if (t == typeof(string)) d_type = myPropInfoType.String; else if (t == typeof(bool) || t == typeof(bool?)) d_type = myPropInfoType.Bool; else if (t == typeof(DateTime) || t == typeof(DateTime?)) d_type = myPropInfoType.DateTime; else if (t.IsEnum) d_type = myPropInfoType.Enum; else if (t == typeof(Guid) || t == typeof(Guid?)) d_type = myPropInfoType.Guid; else if (t == typeof(StringDictionary)) d_type = myPropInfoType.StringDictionary; else if (t == typeof(NameValueCollection)) d_type = myPropInfoType.NameValue; else if (t.IsArray) { d.bt = t.GetElementType(); if (t == typeof(byte[])) d_type = myPropInfoType.ByteArray; else d_type = myPropInfoType.Array; } else if (t.Name.Contains("Dictionary")) { d.GenericTypes = Reflection.Instance.GetGenericArguments(t); if (d.GenericTypes.Length > 0 && d.GenericTypes[0] == typeof(string)) d_type = myPropInfoType.StringKeyDictionary; else d_type = myPropInfoType.Dictionary; } #if !SILVERLIGHT else if (t == typeof(Hashtable)) d_type = myPropInfoType.Hashtable; else if (t == typeof(DataSet)) d_type = myPropInfoType.DataSet; else if (t == typeof(DataTable)) d_type = myPropInfoType.DataTable; #endif else if (IsTypeRegistered(t)) d_type = myPropInfoType.Custom; if (t.IsValueType && !t.IsPrimitive && !t.IsEnum && t != typeof(decimal)) d.IsStruct = true; d.IsInterface = t.IsInterface; d.IsClass = t.IsClass; d.IsValueType = t.IsValueType; if (t.IsGenericType) { d.IsGenericType = true; d.bt = Reflection.Instance.GetGenericArguments(t)[0]; } d.pt = t; d.Name = name; d.changeType = GetChangeType(t); d.Type = d_type; return d; } private Type GetChangeType(Type conversionType) { if (conversionType.IsGenericType && conversionType.GetGenericTypeDefinition().Equals(typeof(Nullable<>))) return Reflection.Instance.GetGenericArguments(conversionType)[0]; return conversionType; } #region [ PROPERTY GET SET ] public string GetTypeAssemblyName(Type t) { string val = ""; if (_tyname.TryGetValue(t, out val)) return val; else { string s = t.AssemblyQualifiedName; _tyname.Add(t, s); return s; } } internal Type GetTypeFromCache(string typename, bool blacklistChecking) { Type val = null; if (_typecache.TryGetValue(typename, out val)) return val; else { // check for BLACK LIST types -> more secure when using $type if (blacklistChecking) { var tn = typename.Trim().ToLowerInvariant(); foreach (var s in _blacklistTypes) if (tn.StartsWith(s, StringComparison.Ordinal)) throw new Exception("Black list type encountered, possible attack vector when using $type : " + typename); } Type t = Type.GetType(typename); #if NET4 if (RDBMode) { if (t == null) // RaptorDB : loading runtime assemblies { t = Type.GetType(typename, (name) => { return AppDomain.CurrentDomain.GetAssemblies().Where(z => z.FullName == name.FullName).FirstOrDefault(); }, null, true); } } #endif _typecache.Add(typename, t); return t; } } internal object FastCreateList(Type objtype, int capacity) { try { int count = 10; if (capacity > 10) count = capacity; CreateList c = null; if (_conlistcache.TryGetValue(objtype, out c)) { if (c != null) // kludge : non capacity lists return c(count); else return FastCreateInstance(objtype); } else { var cinfo = objtype.GetConstructor(new Type[] { typeof(int) }); if (cinfo != null) { DynamicMethod dynMethod = new DynamicMethod("_fcil", objtype, new Type[] { typeof(int) }, true); ILGenerator ilGen = dynMethod.GetILGenerator(); ilGen.Emit(OpCodes.Ldarg_0); ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(new Type[] { typeof(int) })); ilGen.Emit(OpCodes.Ret); c = (CreateList)dynMethod.CreateDelegate(typeof(CreateList)); _conlistcache.Add(objtype, c); return c(count); } else { _conlistcache.Add(objtype, null);// kludge : non capacity lists return FastCreateInstance(objtype); } } } catch (Exception exc) { throw new Exception(string.Format("Failed to fast create instance for type '{0}' from assembly '{1}'", objtype.FullName, objtype.AssemblyQualifiedName), exc); } } internal object FastCreateInstance(Type objtype) { try { CreateObject c = null; if (_constrcache.TryGetValue(objtype, out c)) { return c(); } else { if (objtype.IsClass) { DynamicMethod dynMethod = new DynamicMethod("_fcic", objtype, null, true); ILGenerator ilGen = dynMethod.GetILGenerator(); ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(Type.EmptyTypes)); ilGen.Emit(OpCodes.Ret); c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject)); _constrcache.Add(objtype, c); } else // structs { DynamicMethod dynMethod = new DynamicMethod("_fcis", typeof(object), null, true); ILGenerator ilGen = dynMethod.GetILGenerator(); var lv = ilGen.DeclareLocal(objtype); ilGen.Emit(OpCodes.Ldloca_S, lv); ilGen.Emit(OpCodes.Initobj, objtype); ilGen.Emit(OpCodes.Ldloc_0); ilGen.Emit(OpCodes.Box, objtype); ilGen.Emit(OpCodes.Ret); c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject)); _constrcache.Add(objtype, c); } return c(); } } catch (Exception exc) { throw new Exception(string.Format("Failed to fast create instance for type '{0}' from assembly '{1}'", objtype.FullName, objtype.AssemblyQualifiedName), exc); } } internal static GenericSetter CreateSetField(Type type, FieldInfo fieldInfo) { Type[] arguments = new Type[2]; arguments[0] = arguments[1] = typeof(object); DynamicMethod dynamicSet = new DynamicMethod("_csf", typeof(object), arguments, type, true); ILGenerator il = dynamicSet.GetILGenerator(); if (!type.IsClass) // structs { var lv = il.DeclareLocal(type); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Unbox_Any, type); il.Emit(OpCodes.Stloc_0); il.Emit(OpCodes.Ldloca_S, lv); il.Emit(OpCodes.Ldarg_1); if (fieldInfo.FieldType.IsClass) il.Emit(OpCodes.Castclass, fieldInfo.FieldType); else il.Emit(OpCodes.Unbox_Any, fieldInfo.FieldType); il.Emit(OpCodes.Stfld, fieldInfo); il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Box, type); il.Emit(OpCodes.Ret); } else { il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldarg_1); if (fieldInfo.FieldType.IsValueType) il.Emit(OpCodes.Unbox_Any, fieldInfo.FieldType); il.Emit(OpCodes.Stfld, fieldInfo); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ret); } return (GenericSetter)dynamicSet.CreateDelegate(typeof(GenericSetter)); } internal static FieldInfo GetGetterBackingField(PropertyInfo autoProperty) { var getMethod = autoProperty.GetGetMethod(); // Restrict operation to auto properties to avoid risking errors if a getter does not contain exactly one field read instruction (such as with calculated properties). if (!getMethod.IsDefined(typeof(System.Runtime.CompilerServices.CompilerGeneratedAttribute), false)) return null; var byteCode = getMethod.GetMethodBody()?.GetILAsByteArray() ?? new byte[0]; //var byteCode = getMethod.GetMethodBody().GetILAsByteArray(); int pos = 0; // Find the first LdFld instruction and parse its operand to a FieldInfo object. while (pos < byteCode.Length) { // Read and parse the OpCode (it can be 1 or 2 bytes in size). byte code = byteCode[pos++]; if (!(TryGetOpCode(code, out var opCode) || pos < byteCode.Length && TryGetOpCode((short)(code * 0x100 + byteCode[pos++]), out opCode))) throw new NotSupportedException("Unknown IL code detected."); // If it is a LdFld, read its operand, parse it to a FieldInfo and return it. if (opCode == OpCodes.Ldfld && opCode.OperandType == OperandType.InlineField && pos + sizeof(int) <= byteCode.Length) { return getMethod.Module.ResolveMember(BitConverter.ToInt32(byteCode, pos), getMethod.DeclaringType?.GetGenericArguments(), null) as FieldInfo; } // Otherwise, set the current position to the start of the next instruction, if any (we need to know how much bytes are used by operands). pos += opCode.OperandType == OperandType.InlineNone ? 0 : opCode.OperandType == OperandType.ShortInlineBrTarget || opCode.OperandType == OperandType.ShortInlineI || opCode.OperandType == OperandType.ShortInlineVar ? 1 : opCode.OperandType == OperandType.InlineVar ? 2 : opCode.OperandType == OperandType.InlineI8 || opCode.OperandType == OperandType.InlineR ? 8 : opCode.OperandType == OperandType.InlineSwitch ? 4 * (BitConverter.ToInt32(byteCode, pos) + 1) : 4; } return null; } internal static GenericSetter CreateSetMethod(Type type, PropertyInfo propertyInfo, bool ShowReadOnlyProperties) { MethodInfo setMethod = propertyInfo.GetSetMethod(ShowReadOnlyProperties); if (setMethod == null) { if (!ShowReadOnlyProperties) return null; // If the property has no setter and it is an auto property, try and create a setter for its backing field instead var fld = GetGetterBackingField(propertyInfo); return fld != null ? CreateSetField(type, fld) : null; } Type[] arguments = new Type[2]; arguments[0] = arguments[1] = typeof(object); DynamicMethod setter = new DynamicMethod("_csm", typeof(object), arguments, true);// !setMethod.IsPublic); // fix: skipverify ILGenerator il = setter.GetILGenerator(); if (!type.IsClass) // structs { var lv = il.DeclareLocal(type); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Unbox_Any, type); il.Emit(OpCodes.Stloc_0); il.Emit(OpCodes.Ldloca_S, lv); il.Emit(OpCodes.Ldarg_1); if (propertyInfo.PropertyType.IsClass) il.Emit(OpCodes.Castclass, propertyInfo.PropertyType); else il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType); il.EmitCall(OpCodes.Call, setMethod, null); il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Box, type); } else { if (!setMethod.IsStatic) { il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Castclass, propertyInfo.DeclaringType); il.Emit(OpCodes.Ldarg_1); if (propertyInfo.PropertyType.IsClass) il.Emit(OpCodes.Castclass, propertyInfo.PropertyType); else il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType); il.EmitCall(OpCodes.Callvirt, setMethod, null); il.Emit(OpCodes.Ldarg_0); } else { il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldarg_1); if (propertyInfo.PropertyType.IsClass) il.Emit(OpCodes.Castclass, propertyInfo.PropertyType); else il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType); il.Emit(OpCodes.Call, setMethod); } } il.Emit(OpCodes.Ret); return (GenericSetter)setter.CreateDelegate(typeof(GenericSetter)); } internal static GenericGetter CreateGetField(Type type, FieldInfo fieldInfo) { DynamicMethod dynamicGet = new DynamicMethod("_cgf", typeof(object), new Type[] { typeof(object) }, type, true); ILGenerator il = dynamicGet.GetILGenerator(); if (!type.IsClass) // structs { var lv = il.DeclareLocal(type); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Unbox_Any, type); il.Emit(OpCodes.Stloc_0); il.Emit(OpCodes.Ldloca_S, lv); il.Emit(OpCodes.Ldfld, fieldInfo); if (fieldInfo.FieldType.IsValueType) il.Emit(OpCodes.Box, fieldInfo.FieldType); } else { il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldfld, fieldInfo); if (fieldInfo.FieldType.IsValueType) il.Emit(OpCodes.Box, fieldInfo.FieldType); } il.Emit(OpCodes.Ret); return (GenericGetter)dynamicGet.CreateDelegate(typeof(GenericGetter)); } internal static GenericGetter CreateGetMethod(Type type, PropertyInfo propertyInfo) { MethodInfo getMethod = propertyInfo.GetGetMethod(); if (getMethod == null) return null; DynamicMethod getter = new DynamicMethod("_cgm", typeof(object), new Type[] { typeof(object) }, type, true); ILGenerator il = getter.GetILGenerator(); if (!type.IsClass) // structs { var lv = il.DeclareLocal(type); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Unbox_Any, type); il.Emit(OpCodes.Stloc_0); il.Emit(OpCodes.Ldloca_S, lv); il.EmitCall(OpCodes.Call, getMethod, null); if (propertyInfo.PropertyType.IsValueType) il.Emit(OpCodes.Box, propertyInfo.PropertyType); } else { if (!getMethod.IsStatic) { il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Castclass, propertyInfo.DeclaringType); il.EmitCall(OpCodes.Callvirt, getMethod, null); } else il.Emit(OpCodes.Call, getMethod); if (propertyInfo.PropertyType.IsValueType) il.Emit(OpCodes.Box, propertyInfo.PropertyType); } il.Emit(OpCodes.Ret); return (GenericGetter)getter.CreateDelegate(typeof(GenericGetter)); } public Getters[] GetGetters(Type type, /*bool ShowReadOnlyProperties,*/ List IgnoreAttributes) { Getters[] val = null; if (_getterscache.TryGetValue(type, out val)) return val; var bf = BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static; //if (ShowReadOnlyProperties) // bf |= BindingFlags.NonPublic; PropertyInfo[] props = type.GetProperties(bf); List getters = new List(); foreach (PropertyInfo p in props) { bool read_only = false; if (p.GetIndexParameters().Length > 0) {// Property is an indexer continue; } if (!p.CanWrite)// && (ShowReadOnlyProperties == false))//|| isAnonymous == false)) read_only = true; //continue; if (IgnoreAttributes != null) { bool found = false; foreach (var ignoreAttr in IgnoreAttributes) { if (p.IsDefined(ignoreAttr, false)) { found = true; break; } } if (found) continue; } string mName = null; #if NET4 var att = p.GetCustomAttributes(true); foreach (var at in att) { if (at is DataMemberAttribute) { var dm = (DataMemberAttribute)at; if (dm.Name != "") { mName = dm.Name; } } } #endif GenericGetter g = CreateGetMethod(type, p); if (g != null) getters.Add(new Getters { Getter = g, Name = p.Name, lcName = p.Name.ToLowerInvariant(), memberName = mName, ReadOnly = read_only }); } FieldInfo[] fi = type.GetFields(bf); foreach (var f in fi) { bool read_only = false; if (f.IsInitOnly) // && (ShowReadOnlyProperties == false))//|| isAnonymous == false)) read_only = true;//continue; if (IgnoreAttributes != null) { bool found = false; foreach (var ignoreAttr in IgnoreAttributes) { if (f.IsDefined(ignoreAttr, false)) { found = true; break; } } if (found) continue; } string mName = null; #if NET4 var att = f.GetCustomAttributes(true); foreach (var at in att) { if (at is DataMemberAttribute) { var dm = (DataMemberAttribute)at; if (dm.Name != "") { mName = dm.Name; } } } #endif if (f.IsLiteral == false) { GenericGetter g = CreateGetField(type, f); if (g != null) getters.Add(new Getters { Getter = g, Name = f.Name, lcName = f.Name.ToLowerInvariant(), memberName = mName, ReadOnly = read_only }); } } val = getters.ToArray(); _getterscache.Add(type, val); return val; } //private static bool IsAnonymousType(Type type) //{ // // may break in the future if compiler defined names change... // const string CS_ANONYMOUS_PREFIX = "<>f__AnonymousType"; // const string VB_ANONYMOUS_PREFIX = "VB$AnonymousType"; // if (type == null) // throw new ArgumentNullException("type"); // if (type.Name.StartsWith(CS_ANONYMOUS_PREFIX, StringComparison.Ordinal) || type.Name.StartsWith(VB_ANONYMOUS_PREFIX, StringComparison.Ordinal)) // { // return type.IsDefined(typeof(CompilerGeneratedAttribute), false); // } // return false; //} #endregion internal void ResetPropertyCache() { _propertycache = new SafeDictionary>(); } internal void ClearReflectionCache() { _tyname = new SafeDictionary(10); _typecache = new SafeDictionary(10); _constrcache = new SafeDictionary(10); _getterscache = new SafeDictionary(10); _propertycache = new SafeDictionary>(10); _genericTypes = new SafeDictionary(10); _genericTypeDef = new SafeDictionary(10); } } } ================================================ FILE: RaptorDB.Common/fastJSON/dynamic.cs ================================================ #if NET4 using System; using System.Collections; using System.Collections.Generic; using System.Dynamic; using System.Linq; namespace fastJSON { internal class DynamicJson : DynamicObject, IEnumerable { private IDictionary _dictionary { get; set; } private List _list { get; set; } public DynamicJson(string json) { var parse = fastJSON.JSON.Parse(json); if (parse is IDictionary) _dictionary = (IDictionary)parse; else _list = (List)parse; } private DynamicJson(object dictionary) { if (dictionary is IDictionary) _dictionary = (IDictionary)dictionary; } public override IEnumerable GetDynamicMemberNames() { return _dictionary.Keys.ToList(); } public override bool TryGetIndex(GetIndexBinder binder, Object[] indexes, out Object result) { var index = indexes[0]; if (index is int) { result = _list[(int) index]; } else { result = _dictionary[(string) index]; } if (result is IDictionary) result = new DynamicJson(result as IDictionary); return true; } public override bool TryGetMember(GetMemberBinder binder, out object result) { if (_dictionary.TryGetValue(binder.Name, out result) == false) if (_dictionary.TryGetValue(binder.Name.ToLowerInvariant(), out result) == false) return false;// throw new Exception("property not found " + binder.Name); if (result is IDictionary) { result = new DynamicJson(result as IDictionary); } else if (result is List) { List list = new List(); foreach (object item in (List)result) { if (item is IDictionary) list.Add(new DynamicJson(item as IDictionary)); else list.Add(item); } result = list; } return _dictionary.ContainsKey(binder.Name); } IEnumerator IEnumerable.GetEnumerator() { foreach(var o in _list) { yield return new DynamicJson(o as IDictionary); } } } } #endif ================================================ FILE: RaptorDBCore/RaptorDB/RaptorDB.csproj ================================================ netstandard2.0 true ..\..\raptordb.snk netstandard2.0 3.3.19.1 3.3.19.1 3.3.19 ..\..\output true true ResXFileCodeGenerator ================================================ FILE: RaptorDBCore/RaptorDb.Common/RaptorDb.Common.csproj ================================================ netstandard2.0 true ..\..\raptordb.snk 3.3.19.1 3.3.19.1 3.3.19 true ..\..\output\ true TRACE;NET4 ================================================ FILE: RaptorDBCore/test/sample.cs ================================================ // ref : ..\output\raptordb.dll // ref : ..\output\raptordb.common.dll // ref : ..\faker.dll using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using RaptorDB; using RaptorDB.Common; namespace rdbtest { #region [ entities ] public class LineItem { public decimal QTY { get; set; } public string Product { get; set; } public decimal Price { get; set; } public decimal Discount { get; set; } } public class SalesInvoice { public SalesInvoice() { ID = Guid.NewGuid(); } public Guid ID { get; set; } public string CustomerName { get; set; } public string NoCase { get; set; } public string Address { get; set; } public List Items { get; set; } public DateTime Date { get; set; } public int Serial { get; set; } public byte Status { get; set; } public bool Approved { get; set; } } #endregion #region [ view definition ] public class SalesInvoiceViewRowSchema : RDBSchema { public string CustomerName; public string NoCase; public DateTime Date; public string Address; public int Serial; public byte Status; public bool? Approved; } [RegisterView] public class SalesInvoiceView : View { public SalesInvoiceView() { this.Name = "SalesInvoice"; this.Description = "A primary view for SalesInvoices"; this.isPrimaryList = true; this.isActive = true; this.BackgroundIndexing = true; this.Version = 3; this.Schema = typeof(SalesInvoiceViewRowSchema); this.FullTextColumns.Add("CustomerName"); this.FullTextColumns.Add("Address"); this.Mapper = (api, docid, doc) => { //if (doc.Status == 0) // return; api.EmitObject(docid, doc); }; } } #endregion class Program { static RaptorDB.RaptorDB rdb; // 1 instance static void Main(string[] args) { rdb = RaptorDB.RaptorDB.Open("data"); // a "data" folder beside the executable RaptorDB.Global.RequirePrimaryView = false; Console.WriteLine("Registering views.."); rdb.RegisterView(new SalesInvoiceView()); DoWork(); Console.WriteLine("press any key..."); Console.ReadKey(); Console.WriteLine("\r\nShutting down..."); rdb.Shutdown(); // explicit shutdown } static void DoWork() { long c = rdb.DocumentCount(); if (c > 0) // not the first time running { var result = rdb.Query(x => x.Serial < 100); // show the rows Console.WriteLine(fastJSON.JSON.ToNiceJSON(result.Rows, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false })); // show the count Console.WriteLine("Query result count = " + result.Count); return; } Console.Write("Inserting 100,000 documents..."); int count = 100000; for (int i = 0; i < count; i++) { var inv = CreateInvoice(i); // save here rdb.Save(inv.ID, inv); } Console.WriteLine("done."); } static SalesInvoice CreateInvoice(int counter) { // new invoice var inv = new SalesInvoice() { Date = Faker.DateTimeFaker.BirthDay(), Serial = counter % 10000, CustomerName = Faker.NameFaker.Name(), NoCase = "Me " + counter % 10, Status = (byte)(counter % 4), Address = Faker.LocationFaker.Street(), Approved = counter % 100 == 0 ? true : false }; // new line items inv.Items = new List(); for (int k = 0; k < 5; k++) inv.Items.Add(new LineItem() { Product = "prod " + k, Discount = 0, Price = 10 + k, QTY = 1 + k }); return inv; } } } ================================================ FILE: RaptorDBCore/test/test.csproj ================================================ Exe netcoreapp2.0 ..\..\Faker.dll ================================================ FILE: RaptorDBServer/Installer.cs ================================================ using System.ComponentModel; using System.Configuration.Install; using System.ServiceProcess; namespace RaptorDBServer { [RunInstaller(true)] public class CustomServiceInstaller : Installer { private ServiceProcessInstaller process; private ServiceInstaller service; public CustomServiceInstaller() { process = new ServiceProcessInstaller(); process.Account = ServiceAccount.LocalSystem; service = new ServiceInstaller(); service.ServiceName = Program.InstallServiceName; Installers.Add(process); Installers.Add(service); } protected override void OnBeforeInstall(System.Collections.IDictionary savedState) { Context.Parameters["assemblypath"] = "\"" + this.GetType().Assembly.Location + "\" -p " + Program.Port + " -f \"" + Program.Path + "\""; base.OnBeforeInstall(savedState); } } } ================================================ FILE: RaptorDBServer/Program.cs ================================================ using System; using System.ServiceProcess; using System.IO; using System.Reflection; using System.Configuration.Install; namespace RaptorDBServer { static class Program { public static string InstallServiceName; public static int Port = 90; public static string Path = ""; /// /// The main entry point for the application. /// static void Main(string[] args) { if (args.Length == 0) { Console.WriteLine(@" Run with : -i install service -u uninstall service -n [default = RaptorDB] -p [default = 90] -f "); return; } string name = "RaptorDB"; string path = Directory.GetCurrentDirectory(); int port = 90; bool install = false; bool uninstall = false; for (int i = 0; i < args.Length; i++) { if (args[i].Trim() == "-i") install = true; if (args[i].Trim() == "-u") uninstall = true; if (args[i].Trim() == "-p") port = int.Parse(args[++i]); if (args[i].Trim() == "-f") path = args[++i].Trim(); if (args[i].Trim() == "-n") name = "RaptorDB - " + args[++i].Trim(); } InstallServiceName = name; Port = port; Path = path; if (install) { if (IsServiceInstalled(name)) { Console.WriteLine(); Console.WriteLine("Service exists : " + name); return; } // Install service ManagedInstallerClass.InstallHelper(new string[] { Assembly.GetExecutingAssembly().Location }); return; } else if (uninstall) { if (IsServiceInstalled(name) == false) return; // Uninstall service ManagedInstallerClass.InstallHelper(new string[] { "/u", Assembly.GetExecutingAssembly().Location }); return; } if (Environment.UserInteractive == false) ServiceBase.Run(new Service1()); else Dostart(); } private static void Dostart() { var _raptor = new RaptorDB.RaptorDBServer(Port, Path); Console.WriteLine("Press Enter to shutdown..."); Console.ReadLine(); _raptor.Shutdown(); } private static bool IsServiceInstalled(string serviceName) { // Get a list of current services ServiceController[] services = ServiceController.GetServices(); // Look for our service foreach (ServiceController service in services) if (String.Compare(serviceName, service.ServiceName, true) == 0) return true; // Return return false; } } } ================================================ FILE: RaptorDBServer/Properties/AssemblyInfo.cs ================================================ using System.Reflection; [assembly: AssemblyTitle("RaptorDBServer")] [assembly: AssemblyDescription("Stand alone server or Windows service loader")] [assembly: AssemblyProduct("RaptorDBServer")] ================================================ FILE: RaptorDBServer/RaptorDBServer.csproj ================================================  Debug x86 8.0.30703 2.0 {3EEB5C76-8216-4013-915D-94402BB320F6} Exe Properties RaptorDBServer RaptorDBServer v4.0 512 publish\ true Disk false Foreground 7 Days false false true 0 1.0.0.%2a false false true AnyCPU true full true ..\Output\server\ DEBUG;TRACE prompt 4 false false AnyCPU pdbonly true bin\Release\ TRACE prompt 4 false true ..\raptordb.snk BuildVersion.cs Component Component Service1.cs Service1.cs Designer {45F6BE30-989A-4749-B6A0-69099C8661F4} RaptorDB False .NET Framework 3.5 SP1 false md "$(SolutionDir)output" copy "$(TargetPath)" "$(SolutionDir)output\$(TargetFileName)" ================================================ FILE: RaptorDBServer/Service1.Designer.cs ================================================ namespace RaptorDBServer { partial class Service1 { /// /// Required designer variable. /// private System.ComponentModel.IContainer components = null; /// /// Clean up any resources being used. /// /// true if managed resources should be disposed; otherwise, false. protected override void Dispose(bool disposing) { if (disposing && (components != null)) { components.Dispose(); } base.Dispose(disposing); } #region Component Designer generated code /// /// Required method for Designer support - do not modify /// the contents of this method with the code editor. /// private void InitializeComponent() { // // Service1 // this.ServiceName = "RaptorDB"; } #endregion } } ================================================ FILE: RaptorDBServer/Service1.cs ================================================ using System.ServiceProcess; using System.IO; namespace RaptorDBServer { public partial class Service1 : ServiceBase { public Service1() { InitializeComponent(); } RaptorDB.RaptorDBServer _raptor; protected override void OnStart(string[] args) { Directory.SetCurrentDirectory(Path.GetDirectoryName(this.GetType().Assembly.Location)); _raptor = new RaptorDB.RaptorDBServer(Program.Port, Program.Path); } protected override void OnStop() { _raptor.Shutdown(); } protected override void OnShutdown() { _raptor.Shutdown(); } } } ================================================ FILE: RaptorDBServer/Service1.resx ================================================  text/microsoft-resx 2.0 System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 False ================================================ FILE: RaptorDBTest.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 14 VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RaptorDB", "RaptorDB\RaptorDB.csproj", "{45F6BE30-989A-4749-B6A0-69099C8661F4}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tests", "testing\tests.csproj", "{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "datagridbinding", "datagridbinding\datagridbinding.csproj", "{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RaptorDB.Common", "RaptorDB.Common\RaptorDB.Common.csproj", "{32331D51-5BE0-41E2-AF1A-9B086C5AE809}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Views", "Views\Views.csproj", "{A1347486-8D54-4E17-8A22-76EFE61BF37B}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RaptorDBServer", "RaptorDBServer\RaptorDBServer.csproj", "{3EEB5C76-8216-4013-915D-94402BB320F6}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|Mixed Platforms = Debug|Mixed Platforms Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU Release|Mixed Platforms = Release|Mixed Platforms Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Any CPU.Build.0 = Debug|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|x86.ActiveCfg = Debug|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Any CPU.Build.0 = Release|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Mixed Platforms.Build.0 = Release|Any CPU {45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|x86.ActiveCfg = Release|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Any CPU.Build.0 = Debug|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|x86.ActiveCfg = Debug|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Any CPU.ActiveCfg = Release|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Any CPU.Build.0 = Release|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Mixed Platforms.Build.0 = Release|Any CPU {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|x86.ActiveCfg = Release|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Any CPU.Build.0 = Debug|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|x86.ActiveCfg = Debug|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Any CPU.ActiveCfg = Release|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Any CPU.Build.0 = Release|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Mixed Platforms.Build.0 = Release|Any CPU {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|x86.ActiveCfg = Release|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Any CPU.Build.0 = Debug|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|x86.ActiveCfg = Debug|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Any CPU.ActiveCfg = Release|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Any CPU.Build.0 = Release|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Mixed Platforms.Build.0 = Release|Any CPU {32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|x86.ActiveCfg = Release|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Any CPU.Build.0 = Debug|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|x86.ActiveCfg = Debug|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Any CPU.ActiveCfg = Release|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Any CPU.Build.0 = Release|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Mixed Platforms.Build.0 = Release|Any CPU {A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|x86.ActiveCfg = Release|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Any CPU.Build.0 = Debug|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|x86.ActiveCfg = Debug|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Any CPU.ActiveCfg = Release|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Any CPU.Build.0 = Release|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Mixed Platforms.Build.0 = Release|Any CPU {3EEB5C76-8216-4013-915D-94402BB320F6}.Release|x86.ActiveCfg = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection EndGlobal ================================================ FILE: RaptorDB_Doc.nuspec ================================================  RaptorDB_doc 4.0.10 RaptorDB Document Database mgholam https://www.codeproject.com/Articles/375413/RaptorDB-The-Document-Store false NoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication, works on Linux) ================================================ FILE: RaptorDbCore.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.27130.2024 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RaptorDb.Common", "RaptorDbCore\RaptorDb.Common\RaptorDb.Common.csproj", "{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RaptorDB", "RaptorDbCore\RaptorDB\RaptorDB.csproj", "{CD626E71-6EE5-4B89-AD51-B09717C203C0}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "test", "RaptorDBCore\test\test.csproj", "{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}" ProjectSection(ProjectDependencies) = postProject {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74} = {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74} {CD626E71-6EE5-4B89-AD51-B09717C203C0} = {CD626E71-6EE5-4B89-AD51-B09717C203C0} EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Debug|Any CPU.Build.0 = Debug|Any CPU {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Release|Any CPU.ActiveCfg = Release|Any CPU {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Release|Any CPU.Build.0 = Release|Any CPU {CD626E71-6EE5-4B89-AD51-B09717C203C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CD626E71-6EE5-4B89-AD51-B09717C203C0}.Debug|Any CPU.Build.0 = Debug|Any CPU {CD626E71-6EE5-4B89-AD51-B09717C203C0}.Release|Any CPU.ActiveCfg = Release|Any CPU {CD626E71-6EE5-4B89-AD51-B09717C203C0}.Release|Any CPU.Build.0 = Release|Any CPU {4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Debug|Any CPU.Build.0 = Debug|Any CPU {4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Release|Any CPU.ActiveCfg = Release|Any CPU {4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {79525774-233F-4ED6-82E2-0A92423403DE} EndGlobalSection EndGlobal ================================================ FILE: Tools/buildversion.ncs ================================================ using System; using System.IO; using System.Windows.Forms; using System.Text.RegularExpressions; namespace pp { public class pp { public static void Main(string[] args) { if(args.Length<1) return; string filename = args[0]; string buildversion = ""; int buildnum = 0; StreamReader sr = new StreamReader(filename); string s = sr.ReadToEnd(); sr.Close(); string buildnumregex = @"\s*//\s*build\s*number\s*=\s*(?\w*)"; string buildverregex = @"\s*//\s*build\s*version\s*=\s*(?[\w.]*)"; Regex mr = new Regex( buildnumregex, RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace ); Match m = mr.Match(s); string ss= m.Groups["num"].Value; if(ss !="") { buildnum = int.Parse(ss); buildnum++; s=mr.Replace(s,"\r\n// build number = "+buildnum.ToString()); } mr = new Regex( buildverregex, RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace ); m = mr.Match(s); ss =m.Groups["vers"].Value; if(ss !="") buildversion = ss; string currbuild = buildversion+"."+buildnum.ToString(); mr = new Regex( @"AssemblyFileVersion\s*\(""(?[\w.]*"")", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace ); m=mr.Match(s); s=mr.Replace(s,"AssemblyFileVersion(\""+ currbuild + "\""); //MessageBox.Show(s); StreamWriter sw = new StreamWriter(args[0]); sw.Write(s); sw.Close(); } } } ================================================ FILE: Views/Class1.cs ================================================ using System; using System.Collections.Generic; using RaptorDB; namespace SampleViews { #region [ class definitions ] //public enum State //{ // Open, // Closed, // Approved //} public class LineItem { public decimal QTY { get; set; } public string Product { get; set; } public decimal Price { get; set; } public decimal Discount { get; set; } } public class SalesInvoice { public SalesInvoice() { ID = Guid.NewGuid(); } public Guid ID { get; set; } public string CustomerName { get; set; } public string NoCase { get; set; } public string Address { get; set; } public List Items { get; set; } public DateTime Date { get; set; } public int Serial { get; set; } public byte Status { get; set; } public bool Approved { get; set; } //public State InvoiceState { get; set; } } #endregion #region [ views ] public class TestData { public Guid id; public string username; public string password; } public class TestSchema : RDBSchema { public string username; public string password; } public class TestView : View { public TestView() { this.Name = "test"; this.Schema = typeof(TestSchema); this.Version = 1; this.isPrimaryList = true; this.isActive = true; this.BackgroundIndexing = true; this.Mapper = (api, docid, doc) => { api.EmitObject(docid, doc); }; } } [RegisterView] public class SalesInvoiceViewRowSchema : RDBSchema { //[FullText] public string CustomerName; [CaseInsensitive] [StringIndexLength(255)] public string NoCase; public DateTime Date; public string Address; public int Serial; public byte Status;//{ get; set; } public bool? Approved;// { get; set; } //public State InvoiceState; } [RegisterView] public class SalesInvoiceView : View { public SalesInvoiceView() { this.Name = "SalesInvoice"; this.Description = "A primary view for SalesInvoices"; this.isPrimaryList = true; this.isActive = true; this.BackgroundIndexing = true; this.Version = 6; //// uncomment the following for transaction mode //this.TransactionMode = true; this.Schema = typeof(SalesInvoiceViewRowSchema); this.FullTextColumns.Add(nameof(SalesInvoiceViewRowSchema.CustomerName));// "customername"); // this or the attribute this.FullTextColumns.Add(nameof(SalesInvoiceViewRowSchema.Address));// "address"); this.CaseInsensitiveColumns.Add(nameof(SalesInvoiceViewRowSchema.NoCase));// "nocase"); // this or the attribute //this.StringIndexLength.Add("nocase", 255); this.Mapper = (api, docid, doc) => { //int c = api.Count("SalesItemRows", "product = \"prod 1\""); if (doc.Serial == 0) api.RollBack(); api.EmitObject(docid, doc); }; } } public class SalesItemRowsViewRowSchema : RDBSchema { public string Product; public decimal QTY; public decimal Price; public decimal Discount; } [RegisterView] public class SalesItemRowsView : View { public SalesItemRowsView() { this.Name = "SalesItemRows"; this.Description = ""; this.isPrimaryList = false; this.isActive = true; this.BackgroundIndexing = true; this.Schema = typeof(SalesItemRowsViewRowSchema); this.Mapper = (api, docid, doc) => { if (doc.Status == 3 && doc.Items != null) foreach (var item in doc.Items) api.EmitObject(docid, item); }; } } public class NewViewRowSchema : RDBSchema { public string Product; public decimal QTY; public decimal Price; public decimal Discount; } [RegisterView] public class newview : View { public newview() { this.Name = "newview"; this.Description = ""; this.isPrimaryList = false; this.isActive = true; this.BackgroundIndexing = true; this.Version = 1; this.Schema = typeof(NewViewRowSchema); this.Mapper = (api, docid, doc) => { if (doc.Status == 3 && doc.Items != null) foreach (var i in doc.Items) api.EmitObject(docid, i); }; } } #endregion } ================================================ FILE: Views/Properties/AssemblyInfo.cs ================================================ using System.Reflection; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("Views")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Views")] [assembly: AssemblyCopyright("Copyright © Microsoft 2012")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("b3b61807-a394-421f-88f3-a340dc93f53b")] // Version information for an assembly consists of the following four values: // // Major Version // Minor Version // Build Number // Revision // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] [assembly: AssemblyVersion("1.0.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] ================================================ FILE: Views/ServerSide.cs ================================================ using System.Collections.Generic; using System.Linq; using RaptorDB.Common; using SampleViews; namespace Views { public class ServerSide { // so the result can be serialized and is not an anonymous type // since this uses fields, derive from the BindableFields for data binding to work public class sumtype : RaptorDB.BindableFields { public string Product; public decimal TotalPrice; public decimal TotalQTY; } public static List Sum_Products_based_on_filter(IRaptorDB rap, string filter) { var q = rap.Query(filter); var res = from x in q.Rows group x by x.Product into g select new sumtype // avoid anonymous types { Product = g.Key, TotalPrice = g.Sum(p => p.Price), TotalQTY = g.Sum(p => p.QTY) }; return res.ToList(); } public static List Sum_Products_based_on_filter_args(IRaptorDB rap, string filter, params object[] args) { if (args != null) { // get args here } var q = rap.Query(filter); var res = from x in q.Rows group x by x.Product into g select new sumtype // avoid anonymous types { Product = g.Key, TotalPrice = g.Sum(p => p.Price), TotalQTY = g.Sum(p => p.QTY) }; return res.ToList(); } } } ================================================ FILE: Views/Views.csproj ================================================  Debug AnyCPU 8.0.30703 2.0 {A1347486-8D54-4E17-8A22-76EFE61BF37B} Library Properties Views Views v4.0 512 true full false bin\Debug\ DEBUG;TRACE prompt 4 false pdbonly true bin\Release\ TRACE prompt 4 false false ..\key.snk {32331D51-5BE0-41E2-AF1A-9B086C5AE809} RaptorDB.Common md "$(SolutionDir)Output\server\Extensions" xcopy /q /y "$(TargetDir)$(TargetFileName)" "$(SolutionDir)Output\server\Extensions\" ================================================ FILE: WebStudio/README.md ================================================ # RaptorDB WebStudio ## Get started Install the dependencies... ```bash npm install ``` ...then start [Rollup](https://rollupjs.org): ```bash npm run dev ``` Navigate to [localhost:5000](http://localhost:5000). You should see your app running. Edit a component file in `src`, save it, and reload the page to see your changes. ================================================ FILE: WebStudio/build.cmd ================================================ md dist md public copy src\index.html public\ copy src\index.html dist\ copy src\global.css public\ copy src\global.css dist\ npm run build ================================================ FILE: WebStudio/deploy.cmd ================================================ copy dist\*.* ..\raptordb\web pause ================================================ FILE: WebStudio/package.json ================================================ { "name": "RaptorDB-WebStudio", "version": "1.0.0", "devDependencies": { "npm-run-all": "^4.1.5", "rollup": "^1.10.1", "rollup-plugin-commonjs": "^9.3.4", "rollup-plugin-livereload": "^1.0.0", "rollup-plugin-node-resolve": "^4.2.3", "rollup-plugin-svelte": "^5.0.3", "rollup-plugin-terser": "^4.0.4", "svelte": "^3.0.0" }, "dependencies": { "sirv-cli": "^0.4.4" }, "scripts": { "build": "rollup -c --compact", "autobuild": "rollup -c -w", "dev": "run-p start:dev autobuild", "start": "sirv public --single", "start:dev": "sirv public --single --dev" } } ================================================ FILE: WebStudio/rollup.config.js ================================================ import svelte from 'rollup-plugin-svelte'; import resolve from 'rollup-plugin-node-resolve'; import commonjs from 'rollup-plugin-commonjs'; import livereload from 'rollup-plugin-livereload'; import { terser } from 'rollup-plugin-terser'; const production = !process.env.ROLLUP_WATCH; const dir = production ? "dist" : "public"; export default { input: production ? 'src/main.js' : 'src/debug.js', output: { sourcemap: !production, format: 'iife', name: 'app', file: dir + '/bundle.js' }, plugins: [ svelte({ // enable run-time checks when not in production dev: !production, // we'll extract any component CSS out into // a separate file — better for performance css: css => { css.write(dir + '/bundle.css', !production); // disable sourcemap } }), // If you have external dependencies installed from // npm, you'll most likely need these plugins. In // some cases you'll need additional configuration — // consult the documentation for details: // https://github.com/rollup/rollup-plugin-commonjs resolve({ browser: true }), commonjs(), // Watch the `public` directory and refresh the // browser on changes when not in production !production && livereload(dir), // 'public' // If we're building for production (npm run build // instead of npm run dev), minify production && terser() ], watch: { clearScreen: true } }; ================================================ FILE: WebStudio/src/App.svelte ================================================
showtab(event.detail)} />
    X {#each tabs as tab (tab.id)}
  • changetab(tab.id)}>
    {#if tab.title !== 'Help'} {/if}
    • {/each}
{#if showmsg} (showmsg = false)} {title}>
{#if modalyes} {/if}
{/if} ================================================ FILE: WebStudio/src/UI/Button.svelte ================================================ {#if href} {:else} {/if} ================================================ FILE: WebStudio/src/UI/Modal.svelte ================================================

{title}

================================================ FILE: WebStudio/src/UI/datatable.svelte ================================================ {#if rows !== null && rows.length > 0} 
Rows : {rows.length} of {totalrows.toLocaleString()}
{#each Object.keys(rows[0]) as col (col)} {/each} {#each rows as row} {#each Object.keys(row) as r (r)} {/each} {/each}
sortby(col)}>{col}
{#if r === 'docid'} {:else}{row[r]}{/if}
{:else}

No data

{/if} ================================================ FILE: WebStudio/src/UI/nav.svelte ================================================
logo {#each Object.keys(navdata) as name}
{#each navdata[name] as panel} {/each}
{/each}
================================================ FILE: WebStudio/src/debug.js ================================================ import './main.js' window.ServerURL = "http://localhost:91/"; ================================================ FILE: WebStudio/src/global.css ================================================ body { font-family: sans-serif; color: #222; background: #414141; } .tab-content { background-color: white; padding: 5px; padding-top: 15px; min-height: 90vh; } .LogArea { border-style: solid; border-color: gray; background-color: #e0e0e0; padding: 5px; } .LogItems { font-family: "Courier New", monospace; white-space: pre-wrap; font-size: smaller; } .JSONArea { border-style: solid; border-color: gray; background-color: white; padding: 5px; min-height: 500px; } .JSON { font-family: "Courier New", monospace; white-space: pre-wrap; font-size: smaller; } ================================================ FILE: WebStudio/src/index.html ================================================ RaptorDB WebStudio ================================================ FILE: WebStudio/src/main.js ================================================ import App from './App.svelte'; const app = new App({ target: document.body, }); window.ServerURL = document.location.protocol + "//" + document.location.hostname + ":" + document.location.port + "/"; window.GET = function(url, callback, error) { var xhr = new XMLHttpRequest(); xhr.open('GET', window.ServerURL + url); xhr.onreadystatechange = function() { if (xhr.readyState === 1) { return; } else if (xhr.readyState === 4) { if (xhr.status < 400) { var data = null; if (xhr.responseText !== "") data = JSON.parse(xhr.responseText); else data = ""; if (callback) callback(data); } else if (xhr.status === 401) { xhr.abort(); if (callback) callback(""); } else if (error !== null) error(xhr.responseText); } }; xhr.onerror = function(err) { // $.showDialog("Sorry", "

Connection Failed!

"); console.log(err); }; xhr.withCredentials = false; xhr.send(); } window.LOAD = function(url, callback, error) { var xhr = new XMLHttpRequest(); xhr.open('GET', window.ServerURL + url); xhr.onreadystatechange = function() { if (xhr.readyState === 1) { return; } else if (xhr.readyState === 4) { // console.log(xhr); if (xhr.status < 400) { var data = null; if (xhr.responseText !== "") data = xhr.responseText; else data = ""; if (callback) callback(data); } else if (xhr.status === 401) { xhr.abort(); if (callback) callback(""); } else if (error !== null) error(xhr.responseText); } }; xhr.onerror = function(err) { // $.showDialog("Sorry", "

Connection Failed!

"); console.log(err); }; xhr.withCredentials = false; xhr.send(); } export default app; ================================================ FILE: WebStudio/src/pages/dochistory.svelte ================================================ {#if active}


{#if versions.length > 0}
{#each versions as v (v)} {/each}
{jsondata}
{/if}
{/if} ================================================ FILE: WebStudio/src/pages/docsearch.svelte ================================================ {#if active}
{ if ((event.keyCode || event.which) == 13) docsearch(); }} bind:value={find} />

{/if} ================================================ FILE: WebStudio/src/pages/docview.svelte ================================================ {#if active}
{ if ((event.keyCode || event.which) == 13) docview(); }} bind:value={docid} />

{#if doc !== ''}
{/if}
{/if} ================================================ FILE: WebStudio/src/pages/help.svelte ================================================ {#if active}

Welcome to the RaptorDB Web Studio

You can use this Web interface to query and see configuration information.
{/if} ================================================ FILE: WebStudio/src/pages/hfbrowser.svelte ================================================ {#if active}


{ if ((event.keyCode || event.which) == 13) showkey(key); }} bind:value={key} />

{/if} ================================================ FILE: WebStudio/src/pages/query.svelte ================================================ {#if active}


{#each colnames as col (col)} {/each}