Repository: GrzegorzBlok/FastRsyncNet Branch: master Commit: 961fce59da91 Files: 121 Total size: 375.9 KB Directory structure: gitextract_bd403tkm/ ├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md └── source/ ├── .editorconfig ├── FastRsync/ │ ├── Core/ │ │ ├── AggregateCopyOperationsDecorator.cs │ │ ├── BinaryFormat.cs │ │ ├── ChunkSignature.cs │ │ ├── ChunkSignatureChecksumComparer.cs │ │ ├── JsonContext.cs │ │ ├── JsonSerializationSettings.cs │ │ └── SupportedAlgorithms.cs │ ├── Delta/ │ │ ├── BinaryDeltaReader.cs │ │ ├── BinaryDeltaWriter.cs │ │ ├── DataRange.cs │ │ ├── DeltaApplier.cs │ │ ├── DeltaBuilder.cs │ │ ├── DeltaMetadata.cs │ │ ├── IDeltaReader.cs │ │ └── IDeltaWriter.cs │ ├── Diagnostics/ │ │ ├── ConsoleProgressReporter.cs │ │ └── ProgressReport.cs │ ├── FastRsync.csproj │ ├── Hash/ │ │ ├── Adler32RollingChecksum.cs │ │ ├── Adler32RollingChecksumV2.cs │ │ ├── Adler32RollingChecksumV3.cs │ │ ├── CryptographyHashAlgorithmWrapper.cs │ │ ├── IHashAlgorithm.cs │ │ ├── IRollingChecksum.cs │ │ └── NonCryptographicHashAlgorithmWrapper.cs │ └── Signature/ │ ├── ISignatureReader.cs │ ├── ISignatureWriter.cs │ ├── Signature.cs │ ├── SignatureBuilder.cs │ ├── SignatureReader.cs │ └── SignatureWriter.cs ├── FastRsync.BackwardCompatibilityTests/ │ ├── BackwardCompatibilityTests.cs │ └── FastRsync.BackwardCompatibilityTests.csproj ├── FastRsync.Benchmarks/ │ ├── BuildPatchBenchmark.cs │ ├── FastRsync.Benchmarks.csproj │ ├── HashBenchmark.cs │ ├── Program.cs │ ├── RollingChecksumBenchmark.cs │ └── SignatureBenchmark.cs ├── FastRsync.Compression/ │ ├── FastRsync.Compression.csproj │ └── GZip.cs ├── FastRsync.Tests/ │ ├── Adler32RollingChecksumAlgorithmsTests.cs │ ├── Adler32RollingChecksumTests.cs │ ├── Adler32RollingChecksumV2Tests.cs │ ├── Adler32RollingChecksumV3Tests.cs │ ├── BackwardCompatibilityTests.cs │ ├── CommonAsserts.cs │ ├── DeltaReaderTests.cs │ ├── FastRsync.Tests.csproj │ ├── FastRsyncLegacy/ │ │ ├── BinaryDeltaReaderLegacy.cs │ │ ├── DeltaMetadataLegacy.cs │ │ ├── IDeltaReaderLegacy.cs │ │ └── NewtonsoftJsonSerializationSettings.cs │ ├── GZipTests.cs │ ├── HashTests.cs │ ├── OctodiffLegacy/ │ │ ├── IOctodiffDeltaWriter.cs │ │ ├── IOctodiffSignatureWriter.cs │ │ ├── OctodiffAggregateCopyOperationsDecorator.cs │ │ ├── OctodiffBinaryDeltaWriter.cs │ │ ├── OctodiffBinaryFormat.cs │ │ ├── OctodiffDeltaBuilder.cs │ │ ├── OctodiffSignature.cs │ │ ├── OctodiffSignatureBuilder.cs │ │ └── OctodiffSignatureReader.cs │ ├── PatchingAsyncTests.cs │ ├── PatchingBigFilesTests.cs │ ├── PatchingSyncTests.cs │ ├── SignatureBuilderAsyncRandomDataTests.cs │ ├── SignatureBuilderSyncRandomDataTests.cs │ ├── SignatureBuilderTests.cs │ ├── SignatureReaderTests.cs │ └── Utils.cs ├── FastRsync.sln ├── Octodiff/ │ ├── CommandLine/ │ │ ├── DeltaCommand.cs │ │ ├── ExplainDeltaCommand.cs │ │ ├── HelpCommand.cs │ │ ├── PatchCommand.cs │ │ ├── SignatureCommand.cs │ │ └── Support/ │ │ ├── CommandAttribute.cs │ │ ├── CommandException.cs │ │ ├── CommandLocator.cs │ │ ├── ICommand.cs │ │ ├── ICommandLocator.cs │ │ ├── ICommandMetadata.cs │ │ └── NDesk.Options.cs │ ├── Octodiff.csproj │ ├── OctodiffProgram.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ └── app.config ├── Octodiff.Tests/ │ ├── DeltaFixture.cs │ ├── HelpFixture.cs │ ├── Octodiff.Tests.csproj │ ├── PackageGenerator.cs │ ├── PatchFixture.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── SignatureFixture.cs │ ├── Timings.cs │ └── Util/ │ ├── CommandLineFixture.cs │ └── SilentProcessRunner.cs ├── OctodiffAsync/ │ ├── CommandLine/ │ │ ├── DeltaCommand.cs │ │ ├── ExplainDeltaCommand.cs │ │ ├── HelpCommand.cs │ │ ├── PatchCommand.cs │ │ ├── SignatureCommand.cs │ │ └── Support/ │ │ ├── CommandAttribute.cs │ │ ├── CommandException.cs │ │ ├── CommandLocator.cs │ │ ├── ICommand.cs │ │ ├── ICommandLocator.cs │ │ ├── ICommandMetadata.cs │ │ └── NDesk.Options.cs │ ├── OctodiffAsync.csproj │ ├── OctodiffAsyncProgram.cs │ ├── Properties/ │ │ └── AssemblyInfo.cs │ └── app.config └── Tests.ps1 ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ ############################################################################### # Set default behavior to automatically normalize line endings. ############################################################################### * text=auto ############################################################################### # Set default behavior for command prompt diff. # # This is need for earlier builds of msysgit that does not have it on by # default for csharp files. # Note: This is only used by command line ############################################################################### #*.cs diff=csharp ############################################################################### # Set the merge driver for project and solution files # # Merging from the command prompt will add diff markers to the files if there # are conflicts (Merging from VS is not affected by the settings below, in VS # the diff markers are never inserted). Diff markers may cause the following # file extensions to fail to load in VS. An alternative would be to treat # these files as binary and thus will always conflict and require user # intervention with every merge. To do so, just uncomment the entries below ############################################################################### #*.sln merge=binary #*.csproj merge=binary #*.vbproj merge=binary #*.vcxproj merge=binary #*.vcproj merge=binary #*.dbproj merge=binary #*.fsproj merge=binary #*.lsproj merge=binary #*.wixproj merge=binary #*.modelproj merge=binary #*.sqlproj merge=binary #*.wwaproj merge=binary ############################################################################### # behavior for image files # # image files are treated as binary by default. ############################################################################### #*.jpg binary #*.png binary #*.gif binary ############################################################################### # diff behavior for common document formats # # Convert binary document formats to text before diffing them. This feature # is only available from the command line. Turn it on by uncommenting the # entries below. ############################################################################### #*.doc diff=astextplain #*.DOC diff=astextplain #*.docx diff=astextplain #*.DOCX diff=astextplain #*.dot diff=astextplain #*.DOT diff=astextplain #*.pdf diff=astextplain #*.PDF diff=astextplain #*.rtf diff=astextplain #*.RTF diff=astextplain ================================================ FILE: .gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. # User-specific files *.suo *.user *.sln.docstates # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ x64/ build/ bld/ [Bb]in/ [Oo]bj/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* #NUNIT *.VisualState.xml TestResult.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c *_i.c *_p.c *_i.h *.ilk *.meta *.obj *.pch *.pdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opensdf *.sdf *.cachefile # Visual Studio profiler *.psess *.vsp *.vspx # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # JustCode is a .NET coding addin-in .JustCode # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # NCrunch *.ncrunch* _NCrunch_* .*crunch*.local.xml # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # NuGet Packages Directory packages/ ## TODO: If the tool you use requires repositories.config uncomment the next line #!packages/repositories.config # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets # This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented) !packages/build/ # Windows Azure Build Output csx/ *.build.csdef # Windows Store app package directory AppPackages/ # Others sql/ *.Cache ClientBin/ [Ss]tyle[Cc]op.* ~$* *~ *.dbmdl *.dbproj.schemaview *.pfx *.publishsettings node_modules/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file to a newer # Visual Studio version. Backup files are not needed, because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm # SQL Server files *.mdf *.ldf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings # Microsoft Fakes FakesAssemblies/ /.vs /source/.vs/FastRsync/v15/Server/sqlite3 /source/.vs ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # FastRsyncNet - C# delta syncing library The Fast Rsync .NET library is Rsync implementation derived from [Octodiff](https://github.com/OctopusDeploy/Octodiff) tool. Unlike the Octodiff which is based on SHA1 algorithm, the FastRsyncNet allows a variety of hashing algorithms to choose from. The default one, that is xxHash64, offers significantly faster calculations and smaller signature size than the SHA1, while still providing sufficient quality of hash results. FastRsyncNet supports also SHA1 and is 100% compatible with signatures and deltas produced by Octodiff. Since version 2.0.0 the signature and delta format has changed. FastRsyncNet 2.x is still able to work with signatures and deltas from FastRsync 1.x and Octodiff. However, files made with FastRsyncNet 2.x are not going to be recognized by FastRsyncNet 1.x. ## Install [![NuGet](https://img.shields.io/nuget/v/FastRsyncNet.svg?style=flat)](https://www.nuget.org/packages/FastRsyncNet/) Add To project via NuGet: 1. Right click on a project and click 'Manage NuGet Packages'. 2. Search for 'FastRsyncNet' and click 'Install'. ## Examples ### Calculating signature ```csharp using FastRsync.Signature; ... var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32RollingV3()); using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { signatureBuilder.Build(basisStream, new SignatureWriter(signatureStream)); } ``` ### Calculating delta ```csharp using FastRsync.Delta; ... var delta = new DeltaBuilder(); builder.ProgressReport = new ConsoleProgressReporter(); using (var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { delta.BuildDelta(newFileStream, new SignatureReader(signatureStream, delta.ProgressReporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); } ``` ### Patching (applying delta) ```csharp using FastRsync.Delta; ... var delta = new DeltaApplier { SkipHashCheck = true }; using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var newFileStream = new FileStream(newFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) { delta.Apply(basisStream, new BinaryDeltaReader(deltaStream, progressReporter), newFileStream); } ``` ### Calculating signature on Azure blobs FastRsyncNet might not work on Azure Storage emulator due to issues with stream seeking. ```csharp using FastRsync.Signature; ... var storageAccount = CloudStorageAccount.Parse("azure storage connectionstring"); var blobClient = storageAccount.CreateCloudBlobClient(); var blobsContainer = blobClient.GetContainerReference("containerName"); var basisBlob = blobsContainer.GetBlockBlobReference("blobName"); var signatureBlob = container.GetBlockBlobReference("blob_signature"); var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32RollingV3()); using (var signatureStream = await signatureBlob.OpenWriteAsync()) using (var basisStream = await basisBlob.OpenReadAsync()) { await signatureBuilder.BuildAsync(basisStream, new SignatureWriter(signatureStream)); } ``` ## Available algorithms and relative performance Following signature hashing algorithms are available: * XxHash64 - default algorithm, signature size 6.96 MB, signature calculation time 5209 ms. * XxHash3 - signature size 6.96 MB, signature calculation time 5024 ms. For all new use cases, use this one. * SHA1 - signature size 12.9 MB, signature calculation time 6519 ms. * MD5 - originally used in Rsync program, signature size 10.9 MB, signature calculation time 6767 ms. The signature sizes and calculation times are to provide some insights on relative perfomance. The real perfomance on your system will vary greatly. The benchmark had been run against 0.99 GB file. Following rolling checksum algorithms are available: * Adler32RollingChecksum - default algorithm, it uses low level optimization that makes it faster but provides worse quality of checksum. * Adler32RollingChecksumV2 - Obsolete. It has a bug that - while does not make any data incorrect - results in unnecessary big deltas. Do not use it, unless you need to due to the backward compatibility. It is the original (but incorrectly implemented) Adler32 algorithm implementation (slower but better quality of checksum). * Adler32RollingChecksumV3 - for all new use cases, use this one. It is fast and has best quality of checksum. ## GZip compression that is rsync compatible [![NuGet](https://img.shields.io/nuget/v/FastRsyncNet.Compression.svg?style=flat)](https://www.nuget.org/packages/FastRsyncNet.Compression/) If you synchronize a compressed file, a small change in a compressed file may force rsync algorithm to synchronize whole compressed file, instead of just the changed blocks. To fix this, a custom GZip compression method may be used that periodically reset the compressor state to make it block-sync friendly. Install [FastRsyncNet.Compression](https://www.nuget.org/packages/FastRsyncNet.Compression/) package and use following method: ```csharp FastRsync.Compression.GZip.Compress(Stream sourceStream, Stream destStream) ``` To uncompress you may use any GZip method (e.g. System.IO.Compression.GZipStream). ================================================ FILE: source/.editorconfig ================================================ ; EditorConfig to support per-solution formatting. ; Use the EditorConfig VS add-in to make this work. ; http://editorconfig.org/ ; This is the default for the codeline. root = true [*] end_of_line = CRLF [*.{cs,props,targets,xml,nuspec}] indent_style = space indent_size = 4 [*.{config}] indent_style = space indent_size = 2 [*.{csproj,resx}] indent_style = space indent_size = 2 ================================================ FILE: source/FastRsync/Core/AggregateCopyOperationsDecorator.cs ================================================ using System.IO; using System.Threading; using System.Threading.Tasks; using FastRsync.Delta; namespace FastRsync.Core { // This decorator turns any sequential copy operations into a single operation, reducing // the size of the delta file. // For example: // Copy: 0x0000 - 0x0400 // Copy: 0x0401 - 0x0800 // Copy: 0x0801 - 0x0C00 // Gets turned into: // Copy: 0x0000 - 0x0C00 public class AggregateCopyOperationsDecorator : IDeltaWriter { private readonly IDeltaWriter decorated; private DataRange bufferedCopy; public AggregateCopyOperationsDecorator(IDeltaWriter decorated) { this.decorated = decorated; } public void WriteDataCommand(Stream source, long offset, long length) { FlushCurrentCopyCommand(); decorated.WriteDataCommand(source, offset, length); } public async Task WriteDataCommandAsync(Stream source, long offset, long length, CancellationToken cancellationToken) { FlushCurrentCopyCommand(); await decorated.WriteDataCommandAsync(source, offset, length, cancellationToken).ConfigureAwait(false); } public void WriteMetadata(DeltaMetadata metadata) { decorated.WriteMetadata(metadata); } public void WriteCopyCommand(DataRange chunk) { if (bufferedCopy.Length > 0 && bufferedCopy.StartOffset + bufferedCopy.Length == chunk.StartOffset) { bufferedCopy.Length += chunk.Length; } else { FlushCurrentCopyCommand(); bufferedCopy = chunk; } } private void FlushCurrentCopyCommand() { if (bufferedCopy.Length <= 0) { return; } decorated.WriteCopyCommand(bufferedCopy); bufferedCopy = new DataRange(); } public void Finish() { FlushCurrentCopyCommand(); decorated.Finish(); } } } ================================================ FILE: source/FastRsync/Core/BinaryFormat.cs ================================================ using System.Text; namespace FastRsync.Core { internal class BinaryFormat { public const int SignatureFormatHeaderLength = 7; // OCTOSIG or FRSNCSG public const int DeltaFormatHeaderLength = 9; // OCTODELTA or FRSNCDLTA public const byte CopyCommand = 0x60; public const byte DataCommand = 0x80; } internal class OctoBinaryFormat { public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG"); public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA"); public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>"); public const byte Version = 0x01; } internal class FastRsyncBinaryFormat { public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("FRSNCSG"); public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("FRSNCDLTA"); public const byte Version = 0x01; } } ================================================ FILE: source/FastRsync/Core/ChunkSignature.cs ================================================ using System; namespace FastRsync.Core { public class ChunkSignature { public long StartOffset; // 8 (but not included in the file on disk) public short Length; // 2 public byte[] Hash; // depending on hash (20 for SHA1, 8 for xxHash64) public UInt32 RollingChecksum; // 4 public override string ToString() { return string.Format("{0,6}:{1,6} |{2,20}| {3}", StartOffset, Length, RollingChecksum, BitConverter.ToString(Hash).ToLowerInvariant().Replace("-", "")); } } } ================================================ FILE: source/FastRsync/Core/ChunkSignatureChecksumComparer.cs ================================================ using System.Collections.Generic; namespace FastRsync.Core { class ChunkSignatureChecksumComparer : IComparer { public int Compare(ChunkSignature x, ChunkSignature y) { var comparison = x.RollingChecksum.CompareTo(y.RollingChecksum); return comparison == 0 ? x.StartOffset.CompareTo(y.StartOffset) : comparison; } } } ================================================ FILE: source/FastRsync/Core/JsonContext.cs ================================================ using FastRsync.Delta; using FastRsync.Signature; namespace FastRsync.Core { #if NET7_0_OR_GREATER using System.Text.Json.Serialization; /// /// Provides AOT-safe JSON serialization metadata for FastRsync types. /// [JsonSerializable(typeof(DeltaMetadata))] [JsonSerializable(typeof(SignatureMetadata))] [JsonSourceGenerationOptions( PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true, WriteIndented = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull )] public partial class JsonContextCore : JsonSerializerContext; #endif } ================================================ FILE: source/FastRsync/Core/JsonSerializationSettings.cs ================================================ using System.Text.Json; using System.Text.Json.Serialization; namespace FastRsync.Core { public class JsonSerializationSettings { static JsonSerializationSettings() { JsonSettings = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, PropertyNameCaseInsensitive = true, WriteIndented = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; } public static JsonSerializerOptions JsonSettings { get; } } } ================================================ FILE: source/FastRsync/Core/SupportedAlgorithms.cs ================================================ using System; using System.IO.Hashing; using System.Security.Cryptography; using FastRsync.Hash; namespace FastRsync.Core { public static class SupportedAlgorithms { public static class Hashing { public static IHashAlgorithm Sha1() { return new CryptographyHashAlgorithmWrapper("SHA1", SHA1.Create()); } public static IHashAlgorithm Md5() { return new CryptographyHashAlgorithmWrapper("MD5", MD5.Create()); } public static IHashAlgorithm XxHash() { return new NonCryptographicHashAlgorithmWrapper("XXH64", new XxHash64()); } public static IHashAlgorithm XxHash3() { return new NonCryptographicHashAlgorithmWrapper("XXH3", new XxHash3()); } public static IHashAlgorithm Default() { return XxHash(); } public static IHashAlgorithm Create(string algorithmName) { switch (algorithmName) { case "XXH64": return XxHash(); case "MD5": return Md5(); case "XXH3": return XxHash3(); case "SHA1": return Sha1(); } throw new NotSupportedException($"The hash algorithm '{algorithmName}' is not supported"); } } public static class Checksum { public static IRollingChecksum Adler32Rolling() { return new Adler32RollingChecksum(); } [Obsolete("Adler32V2 has buggy mod operation implemented. See https://github.com/GrzegorzBlok/FastRsyncNet/issues/20")] public static IRollingChecksum Adler32RollingV2() { return new Adler32RollingChecksumV2(); } public static IRollingChecksum Adler32RollingV3() { return new Adler32RollingChecksumV3(); } public static IRollingChecksum Default() { return Adler32Rolling(); } public static IRollingChecksum Create(string algorithm) { switch (algorithm) { case "Adler32": return Adler32Rolling(); case "Adler32V2": return Adler32RollingV2(); case "Adler32V3": return Adler32RollingV3(); default: throw new NotSupportedException($"The rolling checksum algorithm '{algorithm}' is not supported"); } } } } } ================================================ FILE: source/FastRsync/Delta/BinaryDeltaReader.cs ================================================ using System; using System.Collections; using System.IO; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; namespace FastRsync.Delta { public class BinaryDeltaReader : IDeltaReader { private readonly BinaryReader reader; private readonly IProgress progressReport; private byte[] expectedHash; private IHashAlgorithm hashAlgorithm; private readonly int readBufferSize; public BinaryDeltaReader(Stream stream, IProgress progressHandler, int readBufferSize = 4 * 1024 * 1024) { this.reader = new BinaryReader(stream); this.progressReport = progressHandler; this.readBufferSize = readBufferSize; } private DeltaMetadata _metadata; private RsyncFormatType type; public DeltaMetadata Metadata { get { ReadMetadata(); return _metadata; } } public RsyncFormatType Type { get { ReadMetadata(); return type; } } public byte[] ExpectedHash { get { ReadMetadata(); return expectedHash; } } public IHashAlgorithm HashAlgorithm { get { ReadMetadata(); return hashAlgorithm; } } private void ReadMetadata() { if (_metadata != null) return; reader.BaseStream.Seek(0, SeekOrigin.Begin); var header = reader.ReadBytes(BinaryFormat.DeltaFormatHeaderLength); if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.DeltaHeader, header)) { ReadFastRsyncDeltaHeader(); return; } if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.DeltaHeader, header)) { ReadOctoDeltaHeader(); return; } throw new InvalidDataException("The delta file uses a different file format than this program can handle."); } private void ReadFastRsyncDeltaHeader() { var version = reader.ReadByte(); if (version != FastRsyncBinaryFormat.Version) throw new InvalidDataException("The delta file uses a newer file format than this program can handle."); var metadataStr = reader.ReadString(); #if NET7_0_OR_GREATER _metadata = JsonSerializer.Deserialize(metadataStr, JsonContextCore.Default.DeltaMetadata); #else _metadata = JsonSerializer.Deserialize(metadataStr, JsonSerializationSettings.JsonSettings); #endif hashAlgorithm = SupportedAlgorithms.Hashing.Create(_metadata.HashAlgorithm); expectedHash = Convert.FromBase64String(_metadata.ExpectedFileHash); type = RsyncFormatType.FastRsync; } private void ReadOctoDeltaHeader() { var version = reader.ReadByte(); if (version != OctoBinaryFormat.Version) throw new InvalidDataException("The delta file uses a newer file format than this program can handle."); var hashAlgorithmName = reader.ReadString(); hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName); var hashLength = reader.ReadInt32(); expectedHash = reader.ReadBytes(hashLength); var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length); if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta)) throw new InvalidDataException("The delta file appears to be corrupt."); _metadata = new DeltaMetadata { HashAlgorithm = hashAlgorithmName, ExpectedFileHashAlgorithm = hashAlgorithmName, ExpectedFileHash = Convert.ToBase64String(expectedHash) }; type = RsyncFormatType.Octodiff; } public void Apply( Action writeData, Action copy) { var fileLength = reader.BaseStream.Length; ReadMetadata(); while (reader.BaseStream.Position != fileLength) { var b = reader.ReadByte(); progressReport?.Report(new ProgressReport { Operation = ProgressOperationType.ApplyingDelta, CurrentPosition = reader.BaseStream.Position, Total = fileLength }); if (b == BinaryFormat.CopyCommand) { var start = reader.ReadInt64(); var length = reader.ReadInt64(); copy(start, length); } else if (b == BinaryFormat.DataCommand) { var length = reader.ReadInt64(); long soFar = 0; while (soFar < length) { var bytes = reader.ReadBytes((int)Math.Min(length - soFar, readBufferSize)); soFar += bytes.Length; writeData(bytes); } } } } public Task ApplyAsync(Func writeData, Func copy) => ApplyAsync(writeData, copy, CancellationToken.None); public async Task ApplyAsync(Func writeData, Func copy, CancellationToken cancellationToken) { var fileLength = reader.BaseStream.Length; ReadMetadata(); var buffer = new byte[readBufferSize]; while (reader.BaseStream.Position != fileLength) { var b = reader.ReadByte(); progressReport?.Report(new ProgressReport { Operation = ProgressOperationType.ApplyingDelta, CurrentPosition = reader.BaseStream.Position, Total = fileLength }); if (b == BinaryFormat.CopyCommand) { var start = reader.ReadInt64(); var length = reader.ReadInt64(); await copy(start, length).ConfigureAwait(false); } else if (b == BinaryFormat.DataCommand) { var length = reader.ReadInt64(); long soFar = 0; while (soFar < length) { var bytesRead = await reader.BaseStream .ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken) .ConfigureAwait(false); var bytes = buffer; if (bytesRead != buffer.Length) { bytes = new byte[bytesRead]; Array.Copy(buffer, bytes, bytesRead); } soFar += bytes.Length; await writeData(bytes).ConfigureAwait(false); } } } } } } ================================================ FILE: source/FastRsync/Delta/BinaryDeltaWriter.cs ================================================ using System; using System.IO; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; namespace FastRsync.Delta { public class BinaryDeltaWriter : IDeltaWriter { private readonly Stream deltaStream; private readonly BinaryWriter writer; private readonly int readWriteBufferSize; public BinaryDeltaWriter(Stream stream, int readWriteBufferSize = 1024 * 1024) { deltaStream = stream; writer = new BinaryWriter(stream); this.readWriteBufferSize = readWriteBufferSize; } public void WriteMetadata(DeltaMetadata metadata) { writer.Write(FastRsyncBinaryFormat.DeltaHeader); writer.Write(FastRsyncBinaryFormat.Version); #if NET7_0_OR_GREATER var metadataStr = JsonSerializer.Serialize(metadata, JsonContextCore.Default.DeltaMetadata); #else var metadataStr = JsonSerializer.Serialize(metadata, JsonSerializationSettings.JsonSettings); #endif writer.Write(metadataStr); } public void WriteCopyCommand(DataRange segment) { writer.Write(BinaryFormat.CopyCommand); writer.Write(segment.StartOffset); writer.Write(segment.Length); } public void WriteDataCommand(Stream source, long offset, long length) { writer.Write(BinaryFormat.DataCommand); writer.Write(length); var originalPosition = source.Position; try { source.Seek(offset, SeekOrigin.Begin); var buffer = new byte[(int)Math.Min(length, readWriteBufferSize)]; int read; long soFar = 0; while ((read = source.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0) { soFar += read; writer.Write(buffer, 0, read); } } finally { source.Seek(originalPosition, SeekOrigin.Begin); } } public async Task WriteDataCommandAsync(Stream source, long offset, long length, CancellationToken cancellationToken) { writer.Write(BinaryFormat.DataCommand); writer.Write(length); var originalPosition = source.Position; try { source.Seek(offset, SeekOrigin.Begin); var buffer = new byte[(int)Math.Min(length, readWriteBufferSize)]; int read; long soFar = 0; while ((read = await source .ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken) .ConfigureAwait(false)) > 0) { soFar += read; await deltaStream.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false); } } finally { source.Seek(originalPosition, SeekOrigin.Begin); } } public void Finish() { } } } ================================================ FILE: source/FastRsync/Delta/DataRange.cs ================================================ namespace FastRsync.Delta { public struct DataRange { public DataRange(long startOffset, long length) { StartOffset = startOffset; Length = length; } public long StartOffset; public long Length; } } ================================================ FILE: source/FastRsync/Delta/DeltaApplier.cs ================================================ using System; using System.Collections; using System.IO; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; namespace FastRsync.Delta { public class DeltaApplier { private readonly int readBufferSize; public DeltaApplier(int readBufferSize = 4 * 1024 * 1024) { SkipHashCheck = false; this.readBufferSize = readBufferSize; } public bool SkipHashCheck { get; set; } public void Apply(Stream basisFileStream, IDeltaReader delta, Stream outputStream) { var buffer = new byte[readBufferSize]; delta.Apply( writeData: (data) => outputStream.Write(data, 0, data.Length), copy: (startPosition, length) => { basisFileStream.Seek(startPosition, SeekOrigin.Begin); int read; long soFar = 0; while ((read = basisFileStream.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0) { soFar += read; outputStream.Write(buffer, 0, read); } }); if (!SkipHashCheck) { if (!HashCheck(delta, outputStream)) { throw new InvalidDataException( $"Verification of the patched file failed. The {delta.HashAlgorithm.Name} hash of the patch result file, and the file that was used as input for the delta, do not match. This can happen if the basis file changed since the signatures were calculated."); } } } public Task ApplyAsync(Stream basisFileStream, IDeltaReader delta, Stream outputStream) => ApplyAsync(basisFileStream, delta, outputStream, CancellationToken.None); public async Task ApplyAsync(Stream basisFileStream, IDeltaReader delta, Stream outputStream, CancellationToken cancellationToken) { var buffer = new byte[readBufferSize]; await delta.ApplyAsync( writeData: async (data) => await outputStream.WriteAsync(data, 0, data.Length, cancellationToken).ConfigureAwait(false), copy: async (startPosition, length) => { basisFileStream.Seek(startPosition, SeekOrigin.Begin); int read; long soFar = 0; while ((read = await basisFileStream.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0) { soFar += read; await outputStream.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false); } }, cancellationToken).ConfigureAwait(false); if (!SkipHashCheck) { if (!await HashCheckAsync(delta, outputStream, cancellationToken).ConfigureAwait(false)) { throw new InvalidDataException( $"Verification of the patched file failed. The {delta.Metadata.ExpectedFileHashAlgorithm} hash of the patch result file, and the file that was used as input for the delta, do not match. This can happen if the basis file changed since the signatures were calculated."); } } } public bool HashCheck(IDeltaReader delta, Stream outputStream) { outputStream.Seek(0, SeekOrigin.Begin); var sourceFileHash = delta.ExpectedHash; var algorithm = SupportedAlgorithms.Hashing.Create(delta.Metadata.ExpectedFileHashAlgorithm); var actualHash = algorithm.ComputeHash(outputStream); return StructuralComparisons.StructuralEqualityComparer.Equals(sourceFileHash, actualHash); } public Task HashCheckAsync(IDeltaReader delta, Stream outputStream) => HashCheckAsync(delta, outputStream, CancellationToken.None); public async Task HashCheckAsync(IDeltaReader delta, Stream outputStream, CancellationToken cancellationToken) { outputStream.Seek(0, SeekOrigin.Begin); var sourceFileHash = delta.ExpectedHash; var algorithm = SupportedAlgorithms.Hashing.Create(delta.Metadata.ExpectedFileHashAlgorithm); var actualHash = await algorithm.ComputeHashAsync(outputStream, cancellationToken).ConfigureAwait(false); return StructuralComparisons.StructuralEqualityComparer.Equals(sourceFileHash, actualHash); } } } ================================================ FILE: source/FastRsync/Delta/DeltaBuilder.cs ================================================ using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Signature; namespace FastRsync.Delta { public class DeltaBuilder { private readonly int readBufferSize; public DeltaBuilder(int readBufferSize = 4 * 1024 * 1024) { ProgressReport = null; this.readBufferSize = readBufferSize; } public IProgress ProgressReport { get; set; } public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter) { var newFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5(); newFileStream.Seek(0, SeekOrigin.Begin); var newFileHash = newFileVerificationHashAlgorithm.ComputeHash(newFileStream); newFileStream.Seek(0, SeekOrigin.Begin); var signature = signatureReader.ReadSignature(); var chunks = OrderChunksByChecksum(signature.Chunks); var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize); deltaWriter.WriteMetadata(new DeltaMetadata { HashAlgorithm = signature.HashAlgorithm.Name, ExpectedFileHashAlgorithm = newFileVerificationHashAlgorithm.Name, ExpectedFileHash = Convert.ToBase64String(newFileHash), BaseFileHash = signature.Metadata.BaseFileHash, BaseFileHashAlgorithm = signature.Metadata.BaseFileHashAlgorithm }); var checksumAlgorithm = signature.RollingChecksumAlgorithm; var buffer = new byte[readBufferSize]; long lastMatchPosition = 0; var fileSize = newFileStream.Length; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingDelta, CurrentPosition = 0, Total = fileSize }); while (true) { var startPosition = newFileStream.Position; var read = newFileStream.Read(buffer, 0, buffer.Length); if (read <= 0) break; uint checksum = 0; var remainingPossibleChunkSize = maxChunkSize; for (var i = 0; i < read - minChunkSize + 1; i++) { var readSoFar = startPosition + i; var remainingBytes = read - i; if (remainingBytes < maxChunkSize) { remainingPossibleChunkSize = minChunkSize; } if (i == 0 || remainingBytes < maxChunkSize) { checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize); } else { var remove = buffer[i - 1]; var add = buffer[i + remainingPossibleChunkSize - 1]; checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize); } ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingDelta, CurrentPosition = readSoFar, Total = fileSize }); if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize) continue; if (!chunkMap.TryGetValue(checksum, out var startIndex)) continue; for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++) { var chunk = chunks[j]; var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize); if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash)) { readSoFar += remainingPossibleChunkSize; var missing = readSoFar - lastMatchPosition; if (missing > remainingPossibleChunkSize) { deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize); } deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length)); lastMatchPosition = readSoFar; break; } } } if (read < buffer.Length) { break; } newFileStream.Position = newFileStream.Position - maxChunkSize + 1; } if (newFileStream.Length != lastMatchPosition) { deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition); } deltaWriter.Finish(); } public Task BuildDeltaAsync(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter) => BuildDeltaAsync(newFileStream, signatureReader, deltaWriter, CancellationToken.None); public async Task BuildDeltaAsync(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter, CancellationToken cancellationToken) { var newFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5(); newFileStream.Seek(0, SeekOrigin.Begin); var newFileHash = await newFileVerificationHashAlgorithm.ComputeHashAsync(newFileStream, cancellationToken).ConfigureAwait(false); newFileStream.Seek(0, SeekOrigin.Begin); var signature = signatureReader.ReadSignature(); var chunks = OrderChunksByChecksum(signature.Chunks); var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize); deltaWriter.WriteMetadata(new DeltaMetadata { HashAlgorithm = signature.HashAlgorithm.Name, ExpectedFileHashAlgorithm = newFileVerificationHashAlgorithm.Name, ExpectedFileHash = Convert.ToBase64String(newFileHash), BaseFileHash = signature.Metadata.BaseFileHash, BaseFileHashAlgorithm = signature.Metadata.BaseFileHashAlgorithm }); var checksumAlgorithm = signature.RollingChecksumAlgorithm; var buffer = new byte[readBufferSize]; long lastMatchPosition = 0; var fileSize = newFileStream.Length; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingDelta, CurrentPosition = 0, Total = fileSize }); while (true) { var startPosition = newFileStream.Position; var read = await newFileStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false); if (read <= 0) break; uint checksum = 0; var remainingPossibleChunkSize = maxChunkSize; for (var i = 0; i < read - minChunkSize + 1; i++) { var readSoFar = startPosition + i; var remainingBytes = read - i; if (remainingBytes < maxChunkSize) { remainingPossibleChunkSize = minChunkSize; } if (i == 0 || remainingBytes < maxChunkSize) { checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize); } else { var remove = buffer[i - 1]; var add = buffer[i + remainingPossibleChunkSize - 1]; checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize); } ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingDelta, CurrentPosition = readSoFar, Total = fileSize }); if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize) continue; if (!chunkMap.TryGetValue(checksum, out var startIndex)) continue; for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++) { var chunk = chunks[j]; var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize); if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash)) { readSoFar += remainingPossibleChunkSize; var missing = readSoFar - lastMatchPosition; if (missing > remainingPossibleChunkSize) { await deltaWriter.WriteDataCommandAsync(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize, cancellationToken).ConfigureAwait(false); } deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length)); lastMatchPosition = readSoFar; break; } } } if (read < buffer.Length) { break; } newFileStream.Position = newFileStream.Position - maxChunkSize + 1; } if (newFileStream.Length != lastMatchPosition) { await deltaWriter.WriteDataCommandAsync(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition, cancellationToken).ConfigureAwait(false); } deltaWriter.Finish(); } private static List OrderChunksByChecksum(List chunks) { chunks.Sort(new ChunkSignatureChecksumComparer()); return chunks; } private Dictionary CreateChunkMap(IList chunks, out int maxChunkSize, out int minChunkSize) { ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.CreatingChunkMap, CurrentPosition = 0, Total = chunks.Count }); maxChunkSize = 0; minChunkSize = int.MaxValue; var chunkMap = new Dictionary(); for (var i = 0; i < chunks.Count; i++) { var chunk = chunks[i]; if (chunk.Length > maxChunkSize) { maxChunkSize = chunk.Length; } if (chunk.Length < minChunkSize) { minChunkSize = chunk.Length; } if (!chunkMap.ContainsKey(chunk.RollingChecksum)) { chunkMap[chunk.RollingChecksum] = i; } ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.CreatingChunkMap, CurrentPosition = i, Total = chunks.Count }); } return chunkMap; } } } ================================================ FILE: source/FastRsync/Delta/DeltaMetadata.cs ================================================ namespace FastRsync.Delta { public class DeltaMetadata { public string HashAlgorithm { get; set; } public string ExpectedFileHashAlgorithm { get; set; } public string ExpectedFileHash { get; set; } public string BaseFileHashAlgorithm { get; set; } public string BaseFileHash { get; set; } } } ================================================ FILE: source/FastRsync/Delta/IDeltaReader.cs ================================================ using System; using System.Threading; using System.Threading.Tasks; using FastRsync.Hash; using FastRsync.Signature; namespace FastRsync.Delta { public interface IDeltaReader { byte[] ExpectedHash { get; } IHashAlgorithm HashAlgorithm { get; } DeltaMetadata Metadata { get; } RsyncFormatType Type { get; } void Apply(Action writeData, Action copy); Task ApplyAsync(Func writeData, Func copy); Task ApplyAsync(Func writeData, Func copy, CancellationToken cancellationToken); } } ================================================ FILE: source/FastRsync/Delta/IDeltaWriter.cs ================================================ using System.IO; using System.Threading; using System.Threading.Tasks; using FastRsync.Hash; namespace FastRsync.Delta { public interface IDeltaWriter { void WriteMetadata(DeltaMetadata metadata); void WriteCopyCommand(DataRange segment); void WriteDataCommand(Stream source, long offset, long length); Task WriteDataCommandAsync(Stream source, long offset, long length, CancellationToken cancellationToken); void Finish(); } } ================================================ FILE: source/FastRsync/Diagnostics/ConsoleProgressReporter.cs ================================================ using System; namespace FastRsync.Diagnostics { public class ConsoleProgressReporter : IProgress { private ProgressOperationType currentOperation; private int progressPercentage; public void Report(ProgressReport progress) { var percent = (int)((double)progress.CurrentPosition / progress.Total * 100d + 0.5); if (currentOperation != progress.Operation) { progressPercentage = -1; currentOperation = progress.Operation; } if (progressPercentage != percent && percent % 10 == 0) { progressPercentage = percent; Console.WriteLine("{0}: {1}%", currentOperation, percent); } } } } ================================================ FILE: source/FastRsync/Diagnostics/ProgressReport.cs ================================================ namespace FastRsync.Diagnostics { public enum ProgressOperationType { BuildingSignatures, HashingFile, ReadingSignature, CreatingChunkMap, BuildingDelta, ApplyingDelta } public sealed class ProgressReport { public ProgressOperationType Operation { get; internal set; } public long CurrentPosition { get; internal set; } public long Total { get; internal set; } } } ================================================ FILE: source/FastRsync/FastRsync.csproj ================================================  12.0 net462;netstandard2.0;net7.0;net8.0;net9.0;net10.0 FastRsyncNet Grzegorz Blok https://github.com/GrzegorzBlok/FastRsyncNet true Apache-2.0 2.4.5 2.4.5 2.4.5 sync;rsync;synchronization .NET library for file synchronization based on Rsync algorithm. Optimized for speed and data size to achieve best network performance. You may also want to check [FastRsyncNet.Compression](https://www.nuget.org/packages/FastRsyncNet.Compression). true full false bin\Debug\ DEBUG;TRACE prompt 4 pdbonly true bin\Release\ TRACE prompt 4 ================================================ FILE: source/FastRsync/Hash/Adler32RollingChecksum.cs ================================================ using System; namespace FastRsync.Hash { public class Adler32RollingChecksum : IRollingChecksum { public string Name => "Adler32"; public UInt32 Calculate(byte[] block, int offset, int count) { var a = 1; var b = 0; for (var i = offset; i < offset + count; i++) { var z = block[i]; a = (ushort)(z + a); b = (ushort)(b + a); } return (UInt32) ((b << 16) | a); } public UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize) { var b = (ushort)(checksum >> 16 & 0xffff); var a = (ushort)(checksum & 0xffff); a = (ushort)((a - remove + add)); b = (ushort)((b - (chunkSize * remove) + a - 1)); return (UInt32) ((b << 16) | a); } } } ================================================ FILE: source/FastRsync/Hash/Adler32RollingChecksumV2.cs ================================================ using System; namespace FastRsync.Hash { [Obsolete("Adler32V2 has buggy mod operation implemented. See https://github.com/GrzegorzBlok/FastRsyncNet/issues/20")] public class Adler32RollingChecksumV2 : IRollingChecksum { public string Name => "Adler32V2"; private const ushort Modulus = 65521; public uint Calculate(byte[] block, int offset, int count) { var a = 1; var b = 0; for (var i = offset; i < offset + count; i++) { var z = block[i]; a = (z + a) % Modulus; b = (b + a) % Modulus; } return (uint)((b << 16) | a); } public uint Rotate(uint checksum, byte remove, byte add, int chunkSize) { var b = (ushort)(checksum >> 16 & 0xffff); var a = (ushort)(checksum & 0xffff); a = (ushort)((a - remove + add) % Modulus); b = (ushort)((b - (chunkSize * remove) + a - 1) % Modulus); return (uint)((b << 16) | a); } } } ================================================ FILE: source/FastRsync/Hash/Adler32RollingChecksumV3.cs ================================================ using System.Reflection; using System.Runtime.CompilerServices; namespace FastRsync.Hash { public class Adler32RollingChecksumV3 : IRollingChecksum { public string Name => "Adler32V3"; private const ushort Modulus = 65521; public uint Calculate(byte[] block, int offset, int count) { var a = 1; var b = 0; for (count += offset; offset < count; ++offset) { a += block[offset]; a -= a >= Modulus ? Modulus : 0; b += a; b -= b >= Modulus ? Modulus : 0; } return (uint)((b << 16) | a); } public uint Rotate(uint checksum, byte remove, byte add, int chunkSize) { var b = (ushort)(checksum >> 16); var a = (ushort)checksum; var temp = a - remove + add; a = (ushort)(temp < 0 ? temp + Modulus : temp >= Modulus ? temp - Modulus : temp); temp = (b - (chunkSize * remove) + a - 1) % Modulus; b = (ushort)(temp < 0 ? temp + Modulus : temp); return (uint)((b << 16) | a); } } } ================================================ FILE: source/FastRsync/Hash/CryptographyHashAlgorithmWrapper.cs ================================================ using System.IO; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; namespace FastRsync.Hash { public class CryptographyHashAlgorithmWrapper : IHashAlgorithm { private readonly HashAlgorithm algorithm; public CryptographyHashAlgorithmWrapper(string name, HashAlgorithm algorithm) { Name = name; this.algorithm = algorithm; } public string Name { get; } public int HashLengthInBytes => algorithm.HashSize / 8; public byte[] ComputeHash(Stream stream) { return algorithm.ComputeHash(stream); } public async Task ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default) { #if (NET5_0_OR_GREATER) return await algorithm.ComputeHashAsync(stream, cancellationToken).ConfigureAwait(false); #else return await Task.Run(() => algorithm.ComputeHash(stream), cancellationToken).ConfigureAwait(false); #endif } public byte[] ComputeHash(byte[] buffer, int offset, int length) { return algorithm.ComputeHash(buffer, offset, length); } } } ================================================ FILE: source/FastRsync/Hash/IHashAlgorithm.cs ================================================ using System.IO; using System.Threading; using System.Threading.Tasks; namespace FastRsync.Hash { public interface IHashAlgorithm { string Name { get; } int HashLengthInBytes { get; } byte[] ComputeHash(Stream stream); Task ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default); byte[] ComputeHash(byte[] buffer, int offset, int length); } } ================================================ FILE: source/FastRsync/Hash/IRollingChecksum.cs ================================================ using System; namespace FastRsync.Hash { public interface IRollingChecksum { string Name { get; } UInt32 Calculate(byte[] block, int offset, int count); UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize); } } ================================================ FILE: source/FastRsync/Hash/NonCryptographicHashAlgorithmWrapper.cs ================================================ using System; using System.IO.Hashing; using System.IO; using System.Threading; using System.Threading.Tasks; namespace FastRsync.Hash { public class NonCryptographicHashAlgorithmWrapper : IHashAlgorithm { public string Name { get; } public int HashLengthInBytes => algorithm.HashLengthInBytes; private readonly NonCryptographicHashAlgorithm algorithm; public NonCryptographicHashAlgorithmWrapper(string name, NonCryptographicHashAlgorithm algorithm) { Name = name; this.algorithm = algorithm; } public byte[] ComputeHash(Stream stream) { algorithm.Append(stream); var hash = algorithm.GetHashAndReset(); Array.Reverse(hash); return hash; } public async Task ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default) { await algorithm.AppendAsync(stream, cancellationToken).ConfigureAwait(false); var hash = algorithm.GetHashAndReset(); Array.Reverse(hash); return hash; } public byte[] ComputeHash(byte[] buffer, int offset, int length) { var data = buffer.AsSpan(offset, length); algorithm.Append(data); var hash = algorithm.GetHashAndReset(); Array.Reverse(hash); return hash; } } } ================================================ FILE: source/FastRsync/Signature/ISignatureReader.cs ================================================ namespace FastRsync.Signature { public interface ISignatureReader { Signature ReadSignature(); Signature ReadSignatureMetadata(); } } ================================================ FILE: source/FastRsync/Signature/ISignatureWriter.cs ================================================ using System.Threading; using System.Threading.Tasks; using FastRsync.Core; namespace FastRsync.Signature { public interface ISignatureWriter { void WriteMetadata(SignatureMetadata metadata); Task WriteMetadataAsync(SignatureMetadata metadata, CancellationToken cancellationToken); void WriteChunk(ChunkSignature signature); Task WriteChunkAsync(ChunkSignature signature, CancellationToken cancellationToken); } } ================================================ FILE: source/FastRsync/Signature/Signature.cs ================================================ using System.Collections.Generic; using FastRsync.Core; using FastRsync.Hash; namespace FastRsync.Signature { public class SignatureMetadata { public string ChunkHashAlgorithm { get; set; } public string RollingChecksumAlgorithm { get; set; } public string BaseFileHashAlgorithm { get; set; } public string BaseFileHash { get; set; } } public enum RsyncFormatType { Octodiff, FastRsync } public class Signature { public Signature(SignatureMetadata metadata, RsyncFormatType type) { HashAlgorithm = SupportedAlgorithms.Hashing.Create(metadata.ChunkHashAlgorithm); RollingChecksumAlgorithm = SupportedAlgorithms.Checksum.Create(metadata.RollingChecksumAlgorithm); Chunks = new List(); Metadata = metadata; Type = type; } public IHashAlgorithm HashAlgorithm { get; private set; } public IRollingChecksum RollingChecksumAlgorithm { get; private set; } public List Chunks { get; private set; } public SignatureMetadata Metadata { get; private set; } public RsyncFormatType Type { get; private set; } } } ================================================ FILE: source/FastRsync/Signature/SignatureBuilder.cs ================================================ using System; using System.IO; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; namespace FastRsync.Signature { public class SignatureBuilder { public const short MinimumChunkSize = 128; public const short DefaultChunkSize = 2048; public const short MaximumChunkSize = 31 * 1024; private short chunkSize; public SignatureBuilder() : this(SupportedAlgorithms.Hashing.Default(), SupportedAlgorithms.Checksum.Default()) { } public SignatureBuilder(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) { HashAlgorithm = hashAlgorithm; RollingChecksumAlgorithm = rollingChecksumAlgorithm; ChunkSize = DefaultChunkSize; ProgressReport = null; } public IProgress ProgressReport { get; set; } public IHashAlgorithm HashAlgorithm { get; set; } public IRollingChecksum RollingChecksumAlgorithm { get; set; } public short ChunkSize { get => chunkSize; set { if (value < MinimumChunkSize) throw new ArgumentException($"Chunk size cannot be less than {MinimumChunkSize}"); if (value > MaximumChunkSize) throw new ArgumentException($"Chunk size cannot be exceed {MaximumChunkSize}"); chunkSize = value; } } public void Build(Stream baseDataStream, ISignatureWriter signatureWriter) { WriteMetadata(baseDataStream, signatureWriter); WriteChunkSignatures(baseDataStream, signatureWriter); } public Task BuildAsync(Stream baseDataStream, ISignatureWriter signatureWriter) => BuildAsync(baseDataStream, signatureWriter, CancellationToken.None); public async Task BuildAsync(Stream baseDataStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken) { await WriteMetadataAsync(baseDataStream, signatureWriter, cancellationToken).ConfigureAwait(false); await WriteChunkSignaturesAsync(baseDataStream, signatureWriter, cancellationToken).ConfigureAwait(false); } private void WriteMetadata(Stream baseFileStream, ISignatureWriter signatureWriter) { ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.HashingFile, CurrentPosition = 0, Total = baseFileStream.Length }); baseFileStream.Seek(0, SeekOrigin.Begin); var baseFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5(); var baseFileHash = baseFileVerificationHashAlgorithm.ComputeHash(baseFileStream); signatureWriter.WriteMetadata(new SignatureMetadata { ChunkHashAlgorithm = HashAlgorithm.Name, RollingChecksumAlgorithm = RollingChecksumAlgorithm.Name, BaseFileHashAlgorithm = baseFileVerificationHashAlgorithm.Name, BaseFileHash = Convert.ToBase64String(baseFileHash) }); ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.HashingFile, CurrentPosition = baseFileStream.Length, Total = baseFileStream.Length }); } private async Task WriteMetadataAsync(Stream baseFileStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken) { ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.HashingFile, CurrentPosition = 0, Total = baseFileStream.Length }); baseFileStream.Seek(0, SeekOrigin.Begin); var baseFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5(); var baseFileHash = await baseFileVerificationHashAlgorithm.ComputeHashAsync(baseFileStream, cancellationToken).ConfigureAwait(false); await signatureWriter.WriteMetadataAsync(new SignatureMetadata { ChunkHashAlgorithm = HashAlgorithm.Name, RollingChecksumAlgorithm = RollingChecksumAlgorithm.Name, BaseFileHashAlgorithm = baseFileVerificationHashAlgorithm.Name, BaseFileHash = Convert.ToBase64String(baseFileHash) }, cancellationToken).ConfigureAwait(false); ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.HashingFile, CurrentPosition = baseFileStream.Length, Total = baseFileStream.Length }); } private void WriteChunkSignatures(Stream baseFileStream, ISignatureWriter signatureWriter) { var checksumAlgorithm = RollingChecksumAlgorithm; var hashAlgorithm = HashAlgorithm; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingSignatures, CurrentPosition = 0, Total = baseFileStream.Length }); baseFileStream.Seek(0, SeekOrigin.Begin); long start = 0; int read; var block = new byte[ChunkSize]; while ((read = baseFileStream.Read(block, 0, block.Length)) > 0) { signatureWriter.WriteChunk(new ChunkSignature { StartOffset = start, Length = (short)read, Hash = hashAlgorithm.ComputeHash(block, 0, read), RollingChecksum = checksumAlgorithm.Calculate(block, 0, read) }); start += read; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingSignatures, CurrentPosition = start, Total = baseFileStream.Length }); } } private async Task WriteChunkSignaturesAsync(Stream baseFileStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken) { var checksumAlgorithm = RollingChecksumAlgorithm; var hashAlgorithm = HashAlgorithm; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingSignatures, CurrentPosition = 0, Total = baseFileStream.Length }); baseFileStream.Seek(0, SeekOrigin.Begin); long start = 0; int read; var block = new byte[ChunkSize]; while ((read = await baseFileStream.ReadAsync(block, 0, block.Length, cancellationToken).ConfigureAwait(false)) > 0) { await signatureWriter.WriteChunkAsync(new ChunkSignature { StartOffset = start, Length = (short)read, Hash = hashAlgorithm.ComputeHash(block, 0, read), RollingChecksum = checksumAlgorithm.Calculate(block, 0, read) }, cancellationToken).ConfigureAwait(false); start += read; ProgressReport?.Report(new ProgressReport { Operation = ProgressOperationType.BuildingSignatures, CurrentPosition = start, Total = baseFileStream.Length }); } } } } ================================================ FILE: source/FastRsync/Signature/SignatureReader.cs ================================================ using System; using System.Collections; using System.IO; using System.Text.Json; using FastRsync.Core; using FastRsync.Diagnostics; namespace FastRsync.Signature { public class SignatureReader : ISignatureReader { private readonly IProgress report; private readonly BinaryReader reader; public SignatureReader(Stream stream, IProgress progressHandler) { this.report = progressHandler; this.reader = new BinaryReader(stream); } public Signature ReadSignature() { Progress(); var signature = ReadSignatureMetadata(); ReadChunks(signature); return signature; } public Signature ReadSignatureMetadata() { var header = reader.ReadBytes(BinaryFormat.SignatureFormatHeaderLength); if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.SignatureHeader, header)) { return ReadFastRsyncSignatureHeader(); } if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.SignatureHeader, header)) { return ReadOctoSignatureHeader(); } throw new InvalidDataException( "The signature file uses a different file format than this program can handle."); } private Signature ReadFastRsyncSignatureHeader() { var version = reader.ReadByte(); if (version != FastRsyncBinaryFormat.Version) throw new InvalidDataException( "The signature file uses a newer file format than this program can handle."); var metadataStr = reader.ReadString(); #if NET7_0_OR_GREATER var metadata = JsonSerializer.Deserialize(metadataStr, JsonContextCore.Default.SignatureMetadata); #else var metadata = JsonSerializer.Deserialize(metadataStr, JsonSerializationSettings.JsonSettings); #endif var signature = new Signature(metadata, RsyncFormatType.FastRsync); return signature; } private Signature ReadOctoSignatureHeader() { var version = reader.ReadByte(); if (version != OctoBinaryFormat.Version) throw new InvalidDataException( "The signature file uses a newer file format than this program can handle."); var hashAlgorithmName = reader.ReadString(); var rollingChecksumAlgorithmName = reader.ReadString(); var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length); if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta)) throw new InvalidDataException("The signature file appears to be corrupt."); Progress(); var hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName); var rollingChecksumAlgorithm = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithmName); var signature = new Signature(new SignatureMetadata { ChunkHashAlgorithm = hashAlgorithm.Name, RollingChecksumAlgorithm = rollingChecksumAlgorithm.Name }, RsyncFormatType.Octodiff); return signature; } private void ReadChunks(Signature signature) { var expectedHashLength = signature.HashAlgorithm.HashLengthInBytes; long start = 0; var signatureLength = reader.BaseStream.Length; var remainingBytes = signatureLength - reader.BaseStream.Position; var signatureSize = sizeof(ushort) + sizeof(uint) + expectedHashLength; if (remainingBytes % signatureSize != 0) throw new InvalidDataException( "The signature file appears to be corrupt; at least one chunk has data missing."); while (reader.BaseStream.Position < signatureLength - 1) { var length = reader.ReadInt16(); var checksum = reader.ReadUInt32(); var chunkHash = reader.ReadBytes(expectedHashLength); signature.Chunks.Add(new ChunkSignature { StartOffset = start, Length = length, RollingChecksum = checksum, Hash = chunkHash }); start += length; Progress(); } } private void Progress() { report?.Report(new ProgressReport { Operation = ProgressOperationType.ReadingSignature, CurrentPosition = reader.BaseStream.Position, Total = reader.BaseStream.Length }); } } } ================================================ FILE: source/FastRsync/Signature/SignatureWriter.cs ================================================ using System.IO; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FastRsync.Core; namespace FastRsync.Signature { public class SignatureWriter : ISignatureWriter { private readonly Stream signatureStream; private readonly BinaryWriter signaturebw; public SignatureWriter(Stream signatureStream) { this.signatureStream = signatureStream; signaturebw = new BinaryWriter(signatureStream); } private static void WriteMetadataInternal(BinaryWriter bw, SignatureMetadata metadata) { bw.Write(FastRsyncBinaryFormat.SignatureHeader); bw.Write(FastRsyncBinaryFormat.Version); #if NET7_0_OR_GREATER var metadataStr = JsonSerializer.Serialize(metadata, JsonContextCore.Default.SignatureMetadata); #else var metadataStr = JsonSerializer.Serialize(metadata, JsonSerializationSettings.JsonSettings); #endif bw.Write(metadataStr); } public void WriteMetadata(SignatureMetadata metadata) { WriteMetadataInternal(signaturebw, metadata); } public async Task WriteMetadataAsync(SignatureMetadata metadata, CancellationToken cancellationToken) { var ms = new MemoryStream(256); var msbw = new BinaryWriter(ms); WriteMetadataInternal(msbw, metadata); ms.Seek(0, SeekOrigin.Begin); #if (NET5_0_OR_GREATER) await ms.CopyToAsync(signatureStream, cancellationToken).ConfigureAwait(false); #else await ms.CopyToAsync(signatureStream).ConfigureAwait(false); #endif } public void WriteChunk(ChunkSignature signature) { signaturebw.Write(signature.Length); signaturebw.Write(signature.RollingChecksum); signaturebw.Write(signature.Hash); } public async Task WriteChunkAsync(ChunkSignature signature, CancellationToken cancellationToken) { signaturebw.Write(signature.Length); signaturebw.Write(signature.RollingChecksum); await signatureStream.WriteAsync(signature.Hash, 0, signature.Hash.Length, cancellationToken) .ConfigureAwait(false); } } } ================================================ FILE: source/FastRsync.BackwardCompatibilityTests/BackwardCompatibilityTests.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; using NUnit.Framework.Legacy; namespace FastRsync.BackwardCompatibilityTests { [TestFixture] public class BackwardCompatibilityTests { [Test] [TestCase(1378, 129)] [TestCase(13780, 1290)] [TestCase(137800, 1290)] public async Task LegacyLibraryAppliesNewPatch(int baseNumberOfBytes, int newDataNumberOfBytes) { // Arrange - signature and patch from the current library var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false); var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); // Act using the legacy library var progressReporter = Substitute.For>(); var patchedDataStream = new MemoryStream(); var deltaApplier = new FastRsyncLegacy231.Delta.DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new FastRsyncLegacy231.Delta.BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129)] [TestCase(13780, 1290)] [TestCase(137800, 1290)] public async Task LegacyLibraryPreparesAndAppliesNewPatch(int baseNumberOfBytes, int newDataNumberOfBytes) { // Arrange - signature from the current library var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false); // Act using the legacy library var deltaStream = new MemoryStream(); var deltaBuilder = new FastRsyncLegacy231.Delta.DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new FastRsyncLegacy231.Signature.SignatureReader(baseSignatureStream, null), new FastRsyncLegacy231.Core.AggregateCopyOperationsDecorator(new FastRsyncLegacy231.Delta.BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); var progressReporter = Substitute.For>(); var patchedDataStream = new MemoryStream(); var deltaApplier = new FastRsyncLegacy231.Delta.DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new FastRsyncLegacy231.Delta.BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129)] [TestCase(13780, 1290)] [TestCase(137800, 1290)] public async Task LegacyLibraryPreparesPatchForNewLibraryToApplyIt(int baseNumberOfBytes, int newDataNumberOfBytes) { // Arrange - signature from current library, patch from old library var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false); var deltaStream = new MemoryStream(); var deltaBuilder = new FastRsyncLegacy231.Delta.DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new FastRsyncLegacy231.Signature.SignatureReader(baseSignatureStream, null), new FastRsyncLegacy231.Core.AggregateCopyOperationsDecorator(new FastRsyncLegacy231.Delta.BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); // Act - apply patch using the current library var progressReporter = Substitute.For>(); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } private static async Task<(MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream)> PrepareTestDataAsync(int baseNumberOfBytes, int newDataNumberOfBytes, Hash.IRollingChecksum rollingChecksumAlg) { var baseData = new byte[baseNumberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), rollingChecksumAlg); await signatureBuilder.BuildAsync(baseDataStream, new SignatureWriter(baseSignatureStream)).ConfigureAwait(false); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newData = new byte[newDataNumberOfBytes]; new Random().NextBytes(newData); var newDataStream = new MemoryStream(newData); return (baseDataStream, baseSignatureStream, newData, newDataStream); } } } ================================================ FILE: source/FastRsync.BackwardCompatibilityTests/FastRsync.BackwardCompatibilityTests.csproj ================================================  net472 7.3 TestData\FastRsyncLegacy231.dll Never ================================================ FILE: source/FastRsync.Benchmarks/BuildPatchBenchmark.cs ================================================ using System; using System.IO; using BenchmarkDotNet.Attributes; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Signature; namespace FastRsync.Benchmarks { public class BuildPatchBenchmark { [Params(128, 16974)] public int BaseFileSize { get; set; } [Params(16974, 128)] public int NewFileSize { get; set; } private byte[] newFileData; private readonly DeltaBuilder deltaBuilder = new(); private MemoryStream newDataStream; private MemoryStream baseSignatureSha1Stream; private MemoryStream baseSignatureXxHashStream; private MemoryStream baseSignatureXxHash3Stream; private MemoryStream baseSignatureMd5Stream; [GlobalSetup] public void GlobalSetup() { var baseFileBytes = new byte[BaseFileSize]; var rnd = new Random(); rnd.NextBytes(baseFileBytes); newFileData = new byte[NewFileSize]; rnd.NextBytes(newFileData); var baseDataStream = new MemoryStream(baseFileBytes); newDataStream = new MemoryStream(newFileData); { var xxHashSignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32Rolling()); baseSignatureXxHashStream = new MemoryStream(); xxHashSignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureXxHashStream)); } { var xxHash3SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32Rolling()); baseSignatureXxHash3Stream = new MemoryStream(); xxHash3SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureXxHashStream)); } { var sha1SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Adler32Rolling()); baseDataStream.Seek(0, SeekOrigin.Begin); baseSignatureSha1Stream = new MemoryStream(); sha1SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureSha1Stream)); } { var md5SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.Md5(), SupportedAlgorithms.Checksum.Adler32Rolling()); baseDataStream.Seek(0, SeekOrigin.Begin); baseSignatureMd5Stream = new MemoryStream(); md5SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureMd5Stream)); } } [Benchmark] public byte[] BuildPatchXxHash() { newDataStream.Seek(0, SeekOrigin.Begin); baseSignatureXxHashStream.Seek(0, SeekOrigin.Begin); var deltaStream = new MemoryStream(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureXxHashStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); return deltaStream.ToArray(); } [Benchmark] public byte[] BuildPatchXxHash3() { newDataStream.Seek(0, SeekOrigin.Begin); baseSignatureXxHash3Stream.Seek(0, SeekOrigin.Begin); var deltaStream = new MemoryStream(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureXxHash3Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); return deltaStream.ToArray(); } [Benchmark] public byte[] BuildPatchSha1() { newDataStream.Seek(0, SeekOrigin.Begin); baseSignatureSha1Stream.Seek(0, SeekOrigin.Begin); var deltaStream = new MemoryStream(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureSha1Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); return deltaStream.ToArray(); } [Benchmark] public byte[] BuildPatchMd5() { newDataStream.Seek(0, SeekOrigin.Begin); baseSignatureMd5Stream.Seek(0, SeekOrigin.Begin); var deltaStream = new MemoryStream(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureMd5Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); return deltaStream.ToArray(); } } } ================================================ FILE: source/FastRsync.Benchmarks/FastRsync.Benchmarks.csproj ================================================  Exe net10.0 ================================================ FILE: source/FastRsync.Benchmarks/HashBenchmark.cs ================================================ using System; using System.Data.HashFunction.xxHash; using System.IO; using System.IO.Hashing; using System.Threading.Tasks; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; using BenchmarkDotNet.Attributes; namespace FastRsync.Benchmarks { public class HashBenchmark { [Params(128, 16974, 356879)] public int N { get; set; } private byte[] data; private MemoryStream streamdata; private const string AzureBlobStorageConnectionString = "PUT AZURE BLOB STORAGE CONNECTION STRING HERE"; private readonly BlobServiceClient azureBlobServiceClient = new(AzureBlobStorageConnectionString); private BlobContainerClient azureBlobContainerClient; private readonly IxxHash dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig { HashSizeInBits = 64 }); private readonly NonCryptographicHashAlgorithm systemXxHash = new XxHash64(); [GlobalSetup] public async Task GlobalSetup() { data = new byte[N]; new Random().NextBytes(data); var sdata = new byte[N]; new Random().NextBytes(sdata); streamdata = new MemoryStream(sdata); azureBlobContainerClient = azureBlobServiceClient.GetBlobContainerClient("hashbenchmark"); BlockBlobClient azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata"); await azureBlob.UploadAsync(streamdata); } [Benchmark] public byte[] DataXxHash64() { return dataXxHash.ComputeHash(data).Hash; } [Benchmark] public byte[] DataXxHash64MemoryStream() { streamdata.Seek(0, SeekOrigin.Begin); return dataXxHash.ComputeHash(streamdata).Hash; } [Benchmark] public byte[] DataXxHash64AzureBlobStream() { var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata"); using var blobstream = azureBlob.OpenRead(); return dataXxHash.ComputeHash(blobstream).Hash; } [Benchmark] public byte[] SystemXxHash64() { return XxHash64.Hash(data); } [Benchmark] public byte[] SystemXxHash64MemoryStream() { streamdata.Seek(0, SeekOrigin.Begin); systemXxHash.Append(streamdata); return systemXxHash.GetHashAndReset(); } [Benchmark] public byte[] SystemXxHash64AzureBlobStream() { var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata"); using var blobstream = azureBlob.OpenRead(); systemXxHash.Append(blobstream); return systemXxHash.GetHashAndReset(); } [Benchmark] public byte[] SystemXxHash64Reverse() { var systemXxHash64Dest = XxHash64.Hash(data); Array.Reverse(systemXxHash64Dest); return systemXxHash64Dest; } [Benchmark] public byte[] SystemXxHash64AzureBlobStreamReverse() { var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata"); using var blobstream = azureBlob.OpenRead(); systemXxHash.Append(blobstream); var systemXxHash64Dest = systemXxHash.GetHashAndReset(); Array.Reverse(systemXxHash64Dest); return systemXxHash64Dest; } } } ================================================ FILE: source/FastRsync.Benchmarks/Program.cs ================================================ using System.Reflection; using BenchmarkDotNet.Running; namespace FastRsync.Benchmarks { class Program { static void Main(string[] args) { BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args); } } } ================================================ FILE: source/FastRsync.Benchmarks/RollingChecksumBenchmark.cs ================================================ using System; using BenchmarkDotNet.Attributes; using FastRsync.Core; using FastRsync.Hash; namespace FastRsync.Benchmarks { public class RollingCheckSumBenchmark { [Params(128, 16974, 356879)] public int N { get; set; } private byte[] data; private readonly IRollingChecksum adler32RollingAlgorithm = SupportedAlgorithms.Checksum.Adler32Rolling(); private readonly IRollingChecksum adler32RollingV2Algorithm = SupportedAlgorithms.Checksum.Adler32RollingV2(); private readonly IRollingChecksum adler32RollingV3Algorithm = SupportedAlgorithms.Checksum.Adler32RollingV3(); private uint checksum32; private uint checksum32V2; private uint checksum32V3; [GlobalSetup] public void GlobalSetup() { data = new byte[N]; new Random().NextBytes(data); checksum32 = adler32RollingAlgorithm.Calculate(data, 0, data.Length - 1); checksum32V2 = adler32RollingV2Algorithm.Calculate(data, 0, data.Length - 1); checksum32V3 = adler32RollingV3Algorithm.Calculate(data, 0, data.Length - 1); } [Benchmark] public uint Adler32RollingCalculateChecksum() { return adler32RollingAlgorithm.Calculate(data, 0, data.Length); } [Benchmark] public uint Adler32RollingV2CalculateChecksum() { return adler32RollingV2Algorithm.Calculate(data, 0, data.Length); } [Benchmark] public uint Adler32RollingV3CalculateChecksum() { return adler32RollingV3Algorithm.Calculate(data, 0, data.Length); } [Benchmark] public uint Adler32RollingRotateChecksum() { return adler32RollingAlgorithm.Rotate(checksum32, data[0], data[^1], data.Length - 1); } [Benchmark] public uint Adler32RollingV2RotateChecksum() { return adler32RollingV2Algorithm.Rotate(checksum32V2, data[0], data[^1], data.Length - 1); } [Benchmark] public uint Adler32RollingV3RotateChecksum() { return adler32RollingV3Algorithm.Rotate(checksum32V3, data[0], data[^1], data.Length - 1); } } } ================================================ FILE: source/FastRsync.Benchmarks/SignatureBenchmark.cs ================================================ using System; using System.IO; using BenchmarkDotNet.Attributes; using FastRsync.Core; using FastRsync.Signature; using FastRsync.Tests.OctodiffLegacy; namespace FastRsync.Benchmarks { public class SignatureBenchmark { [Params(128, 16974, 356879)] public int BaseFileSize { get; set; } [Params(SignatureBuilder.MinimumChunkSize, SignatureBuilder.DefaultChunkSize, SignatureBuilder.MaximumChunkSize)] public short ChunkSize { get; set; } private byte[] data; private readonly SignatureBuilder xxHashSignatureBuilder = new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32Rolling()); private readonly SignatureBuilder xxHash3SignatureBuilder = new(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32Rolling()); private readonly SignatureBuilder xxHashAdler32V2SignatureBuilder = new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32RollingV2()); private readonly SignatureBuilder sha1SignatureBuilder = new(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Adler32Rolling()); private readonly SignatureBuilder md5SignatureBuilder = new(SupportedAlgorithms.Hashing.Md5(), SupportedAlgorithms.Checksum.Adler32Rolling()); private readonly OctodiffSignatureBuilder xxHashOctodiffSignatureBuilder = new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32Rolling()); private MemoryStream dataStream; [GlobalSetup] public void GlobalSetup() { data = new byte[BaseFileSize]; new Random().NextBytes(data); xxHashSignatureBuilder.ChunkSize = ChunkSize; xxHashAdler32V2SignatureBuilder.ChunkSize = ChunkSize; sha1SignatureBuilder.ChunkSize = ChunkSize; md5SignatureBuilder.ChunkSize = ChunkSize; dataStream = new MemoryStream(data); } [Benchmark] public byte[] SignaturexxHash() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); xxHashSignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream)); return signatureStream.ToArray(); } [Benchmark] public byte[] SignaturexxHashAdler32V2() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); xxHashAdler32V2SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream)); return signatureStream.ToArray(); } [Benchmark] public byte[] SignaturexxHash3() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); xxHash3SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream)); return signatureStream.ToArray(); } [Benchmark] public byte[] SignatureSha1() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); sha1SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream)); return signatureStream.ToArray(); } [Benchmark] public byte[] SignatureMd5() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); md5SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream)); return signatureStream.ToArray(); } [Benchmark] public byte[] OctodiffSignaturexxHash() { dataStream.Seek(0, SeekOrigin.Begin); var signatureStream = new MemoryStream(); xxHashOctodiffSignatureBuilder.Build(dataStream, new OctodiffSignatureWriter(signatureStream)); return signatureStream.ToArray(); } } } ================================================ FILE: source/FastRsync.Compression/FastRsync.Compression.csproj ================================================  netstandard2.0 true Grzegorz Blok FastRsyncNet.Compression Apache-2.0 .NET library for file compression compatible with Rsync algorithm. You may also want to check [FastRsyncNet](https://www.nuget.org/packages/FastRsyncNet). https://github.com/GrzegorzBlok/FastRsyncNet sync;rsync;synchronization;compression 1.1.2 1.1.2 1.1.2 ================================================ FILE: source/FastRsync.Compression/GZip.cs ================================================ // this implementation is influenced by pigz tool by Mark Adler: https://github.com/madler/pigz/blob/master/pigz.c // pigz license: /* This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Mark Adler madler@alumni.caltech.edu */ using System; using System.IO; using System.Threading; using System.Threading.Tasks; using Ionic.Zlib; namespace FastRsync.Compression { public class GZip { const int RSYNCBITS = 12; const uint RSYNCMASK = ((1U << RSYNCBITS) - (uint)1); const uint RSYNCHIT = (RSYNCMASK >> 1); const int BUFFER_SIZE = 32 * 1024; public static void Compress(Stream sourceStream, Stream destStream) { var buffer = new byte[BUFFER_SIZE]; using (var compressor = new GZipStream(destStream, Ionic.Zlib.CompressionMode.Compress, CompressionLevel.BestSpeed, true)) { compressor.LastModified = new DateTime(1970, 1, 1); uint hash = RSYNCHIT; int n; while ((n = sourceStream.Read(buffer, 0, BUFFER_SIZE)) > 0) { for (int i = 0, j = 0; i < n; i++) { hash = ((hash << 1) ^ buffer[i]) & RSYNCMASK; if (hash == RSYNCHIT) { compressor.FlushMode = FlushType.Sync; compressor.Write(buffer, j, i + 1 - j); j = i + 1; } else if (i + 1 == n) { compressor.FlushMode = FlushType.None; compressor.Write(buffer, j, i + 1 - j); } } } } } public static Task CompressAsync(Stream sourceStream, Stream destStream) => CompressAsync(sourceStream, destStream, CancellationToken.None); public static async Task CompressAsync(Stream sourceStream, Stream destStream, CancellationToken cancellationToken) { var buffer = new byte[BUFFER_SIZE]; using (var compressor = new GZipStream(destStream, Ionic.Zlib.CompressionMode.Compress, CompressionLevel.BestSpeed, true)) { compressor.LastModified = new DateTime(1970, 1, 1); uint hash = RSYNCHIT; int n; while ((n = await sourceStream.ReadAsync(buffer, 0, BUFFER_SIZE, cancellationToken).ConfigureAwait(false)) > 0) { for (int i = 0, j = 0; i < n; i++) { hash = ((hash << 1) ^ buffer[i]) & RSYNCMASK; if (hash == RSYNCHIT) { compressor.FlushMode = FlushType.Sync; await compressor.WriteAsync(buffer, j, i + 1 - j, cancellationToken).ConfigureAwait(false); j = i + 1; } else if (i + 1 == n) { compressor.FlushMode = FlushType.None; await compressor.WriteAsync(buffer, j, i + 1 - j, cancellationToken).ConfigureAwait(false); } } } } } } } ================================================ FILE: source/FastRsync.Tests/Adler32RollingChecksumAlgorithmsTests.cs ================================================ using FastRsync.Hash; using NSubstitute; using NUnit.Framework; using NUnit.Framework.Legacy; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; namespace FastRsync.Tests; [TestFixture] public class Adler32RollingChecksumAlgorithmsTests { public static IRollingChecksum[] RollingChecksumAlgorithms => typeof(IRollingChecksum).Assembly.GetTypes() .Where(t => typeof(IRollingChecksum).IsAssignableFrom(t) && t is {IsClass: true, IsAbstract:false} && t.GetConstructor(Type.EmptyTypes) != null && t.GetCustomAttribute() == null) .Select(t => (IRollingChecksum)Activator.CreateInstance(t)) .ToArray(); private static readonly int[] Sizes = [1024 * 1024 / 2, 10 * 1024 * 1024, 20 * 1024 * 1024]; private static readonly int[] ModificationPercentages = [1, 3, 5, 10]; [Test] [TestCaseSource(nameof(RollingChecksumAlgorithms))] public void EveryRollingChecksumAlgorithmCalculatesRotateCorrectly(IRollingChecksum rollingChecksumAlgorithmInstance) { // Arrange const int len = 8 * 1024; const int windowSize = len - 1; var bytes = new byte[len]; new Random().NextBytes(bytes); // Act var rotatedChecksum = rollingChecksumAlgorithmInstance.Rotate( rollingChecksumAlgorithmInstance.Calculate(bytes, 0, windowSize), bytes[0], bytes[^1], windowSize); var checksum = rollingChecksumAlgorithmInstance.Calculate(bytes, 1, windowSize); // Assert Assert.That(rotatedChecksum, Is.EqualTo(checksum)); } public static IEnumerable SignDeltaAndApplyPatchTestCases() { return from size in Sizes from modPercentage in ModificationPercentages from algo in RollingChecksumAlgorithms select new TestCaseData(size, modPercentage, algo); } [Test] [TestCaseSource(nameof(SignDeltaAndApplyPatchTestCases))] public async Task SignDeltaAndApplyPatchWithEveryRollingChecksumAlgorithm(int size, int modificationPercentage, IRollingChecksum algorithm) { //Arrange var bytesToPatchFrom = GetRandomBytes(size); var targetBytes = ModifyBytes(bytesToPatchFrom, modificationPercentage); //Act var progressReporter = Substitute.For>(); //generate signature of target bytes var signatureStream = new MemoryStream(); var signatureBuilder = new Signature.SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), algorithm); await signatureBuilder.BuildAsync(new MemoryStream(bytesToPatchFrom), new Signature.SignatureWriter(signatureStream)); signatureStream.Seek(0, SeekOrigin.Begin); //generate delta of base bytes and target bytes var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(new MemoryStream(targetBytes), new Signature.SignatureReader(signatureStream, progressReporter), new BinaryDeltaWriter(deltaStream)); deltaStream.Seek(0, SeekOrigin.Begin); //apply delta to base bytes var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(new MemoryStream(bytesToPatchFrom), new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream); patchedDataStream.Seek(0, SeekOrigin.Begin); //Assert CollectionAssert.AreEqual(targetBytes, patchedDataStream.ToArray()); //make sure that the delta is reasonably small, allowing for some overhead but ensuring that the rolling checksum is effective Assert.That(deltaStream.Length, Is.LessThanOrEqualTo(2.5 * modificationPercentage / 100.0 * bytesToPatchFrom.Length)); } private static byte[] ModifyBytes(ReadOnlySpan input, int percentage) { Assert.That(percentage, Is.InRange(0, 100)); var length = input.Length; var modified = new byte[length]; input.CopyTo(modified); var bytesToModify = (int)(length * (percentage / 100.0)); if (bytesToModify == 0) return modified; var random = new Random(); var maxStartPosition = length - bytesToModify; var startIndex = maxStartPosition > 0 ? random.Next(maxStartPosition + 1) : 0; // Modify bytes in a continuous range for (var i = startIndex; i < startIndex + bytesToModify && i < length; i++) { modified[i] = (byte)(modified[i] + 1); } return modified; } private static byte[] GetRandomBytes(int size) { var data = new byte[size]; new Random().NextBytes(data); return data; } } ================================================ FILE: source/FastRsync.Tests/Adler32RollingChecksumTests.cs ================================================ using FastRsync.Hash; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class Adler32RollingChecksumTests { [Test] public void Adler32RollingChecksum_CalculatesChecksum() { // Arrange var data1 = "Adler32 checksum test"u8.ToArray(); var data2 = "Fast Rsync Fast Rsync"u8.ToArray(); var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray(); var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray(); var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray(); // Act var checksum1 = new Adler32RollingChecksum().Calculate(data1, 0, data1.Length); var checksum2 = new Adler32RollingChecksum().Calculate(data2, 0, data2.Length); var checksum3 = new Adler32RollingChecksum().Calculate(data3, 0, data3.Length); var checksum4 = new Adler32RollingChecksum().Calculate(data4, 0, data4.Length); var checksum5 = new Adler32RollingChecksum().Calculate(data5, 0, data5.Length); // Assert Assert.That(checksum1, Is.EqualTo(0x4ff907a1)); Assert.That(checksum2, Is.EqualTo(0x5206079b)); //Assert.That(checksum3, Is.EqualTo(0x040f0fc1)); // bug in adler32 implementation https://github.com/OctopusDeploy/Octodiff/issues/16 //Assert.That(checksum4, Is.EqualTo(0x2d10357d)); //Assert.That(checksum5, Is.EqualTo(0xa05ca509)); } } ================================================ FILE: source/FastRsync.Tests/Adler32RollingChecksumV2Tests.cs ================================================ using FastRsync.Hash; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class Adler32RollingChecksumV2Tests { [Test] public void Adler32RollingChecksumV2_CalculatesChecksum() { // Arrange var data1 = "Adler32 checksum test"u8.ToArray(); var data2 = "Fast Rsync Fast Rsync"u8.ToArray(); var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray(); var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray(); var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray(); // Act var checksum1 = new Adler32RollingChecksumV2().Calculate(data1, 0, data1.Length); var checksum2 = new Adler32RollingChecksumV2().Calculate(data2, 0, data2.Length); var checksum3 = new Adler32RollingChecksumV2().Calculate(data3, 0, data3.Length); var checksum4 = new Adler32RollingChecksumV2().Calculate(data4, 0, data4.Length); var checksum5 = new Adler32RollingChecksumV2().Calculate(data5, 0, data5.Length); // Assert Assert.That(checksum1, Is.EqualTo(0x4ff907a1)); Assert.That(checksum2, Is.EqualTo(0x5206079b)); Assert.That(checksum3, Is.EqualTo(0x040f0fc1)); Assert.That(checksum4, Is.EqualTo(0x2d10357d)); Assert.That(checksum5, Is.EqualTo(0xa05ca509)); } } ================================================ FILE: source/FastRsync.Tests/Adler32RollingChecksumV3Tests.cs ================================================ using FastRsync.Hash; using NUnit.Framework; using System; namespace FastRsync.Tests; [TestFixture] public class Adler32RollingChecksumV3Tests { [Test] public void Adler32RollingChecksumV3_CalculatesChecksum() { // Arrange var data1 = "Adler32 checksum test"u8.ToArray(); var data2 = "Fast Rsync Fast Rsync"u8.ToArray(); var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray(); var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray(); var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray(); // Act var checksum1 = new Adler32RollingChecksumV3().Calculate(data1, 0, data1.Length); var checksum2 = new Adler32RollingChecksumV3().Calculate(data2, 0, data2.Length); var checksum3 = new Adler32RollingChecksumV3().Calculate(data3, 0, data3.Length); var checksum4 = new Adler32RollingChecksumV3().Calculate(data4, 0, data4.Length); var checksum5 = new Adler32RollingChecksumV3().Calculate(data5, 0, data5.Length); // Assert Assert.That(checksum1, Is.EqualTo(0x4ff907a1)); Assert.That(checksum2, Is.EqualTo(0x5206079b)); Assert.That(checksum3, Is.EqualTo(0x040f0fc1)); Assert.That(checksum4, Is.EqualTo(0x2d10357d)); Assert.That(checksum5, Is.EqualTo(0xa05ca509)); } [Test] public void Adler32RollingChecksumV3_RotatesChecksum() { //Arrange const int len = 8 * 1024; const int windowSize = len - 1; var bytes = new byte[len]; new Random().NextBytes(bytes); var adler = new Adler32RollingChecksumV3(); // Act var rotatedChecksum = adler.Rotate(adler.Calculate(bytes, 0, windowSize), bytes[0], bytes[^1], windowSize); var checksum = adler.Calculate(bytes, 1, windowSize); // Assert Assert.That(rotatedChecksum, Is.EqualTo(checksum)); } } ================================================ FILE: source/FastRsync.Tests/BackwardCompatibilityTests.cs ================================================ using NUnit.Framework; using System; using System.IO; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using NSubstitute; using NUnit.Framework.Legacy; namespace FastRsync.Tests; [TestFixture] public class BackwardCompatibilityTests { [Test] [TestCase(".\\TestData\\patch112.bin")] [TestCase(".\\TestData\\patch200.bin")] [TestCase(".\\TestData\\patch231.bin")] public async Task ApplyOldPatch(string deltaFileName) { // Arrange const string baseFileName = ".\\TestData\\basefile.bin"; const string baseFile2Name = ".\\TestData\\basefile2.bin"; await using var baseFileStream = new FileStream(baseFileName, FileMode.Open); await using var deltaFileStream = new FileStream(deltaFileName, FileMode.Open); var progressReporter = Substitute.For>(); // Act var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseFileStream, new BinaryDeltaReader(deltaFileStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert var newFileExpectedData = await File.ReadAllBytesAsync(baseFile2Name); CollectionAssert.AreEqual(newFileExpectedData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(".\\TestData\\signature112.bin")] [TestCase(".\\TestData\\signature200.bin")] [TestCase(".\\TestData\\signature231.bin")] public async Task BuildPatchFromOldSignature(string signatureFileName) { // Arrange const string baseFileName = ".\\TestData\\basefile.bin"; const string baseFile2Name = ".\\TestData\\basefile2.bin"; await using var signatureFileStream = new FileStream(signatureFileName, FileMode.Open); var progressReporter = Substitute.For>(); var patchedDataStream = new MemoryStream(); // Act await using (var baseFile2Stream = new FileStream(baseFile2Name, FileMode.Open)) { var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(baseFile2Stream, new SignatureReader(signatureFileStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); await using var baseFileStream = new FileStream(baseFileName, FileMode.Open); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseFileStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); } // Assert var newFileExpectedData = await File.ReadAllBytesAsync(baseFile2Name); CollectionAssert.AreEqual(newFileExpectedData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } } ================================================ FILE: source/FastRsync.Tests/CommonAsserts.cs ================================================ using System.IO; using FastRsync.Hash; using FastRsync.Signature; using NUnit.Framework; namespace FastRsync.Tests; class CommonAsserts { public static void ValidateSignature(Stream signatureStream, IHashAlgorithm hashAlgorithm, string baseFileHash, IRollingChecksum rollingAlgorithm) { signatureStream.Seek(0, SeekOrigin.Begin); var sig = new SignatureReader(signatureStream, null).ReadSignature(); Assert.That(sig.Type, Is.EqualTo(RsyncFormatType.FastRsync)); Assert.That(sig.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name)); Assert.That(sig.Metadata.ChunkHashAlgorithm, Is.EqualTo(hashAlgorithm.Name)); Assert.That(sig.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes)); Assert.That(sig.RollingChecksumAlgorithm.Name, Is.EqualTo(rollingAlgorithm.Name)); Assert.That(sig.Metadata.RollingChecksumAlgorithm, Is.EqualTo(rollingAlgorithm.Name)); Assert.That(sig.Metadata.BaseFileHashAlgorithm, Is.EqualTo("MD5")); Assert.That(sig.Metadata.BaseFileHash, Is.EqualTo(baseFileHash)); } } ================================================ FILE: source/FastRsync.Tests/DeltaReaderTests.cs ================================================ using System.IO; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Signature; using FastRsync.Tests.FastRsyncLegacy; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class DeltaReaderTests { /// /// Metadata without BaseFileHashAlgorithm and BaseFileHash fields /// private static readonly byte[] FastRsyncLegacyMetadataDelta = { 0x46, 0x52, 0x53, 0x4e, 0x43, 0x44, 0x4c, 0x54, 0x41, 0x01, 0x69, 0x7b, 0x22, 0x68, 0x61, 0x73, 0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a, 0x22, 0x58, 0x58, 0x48, 0x36, 0x34, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a, 0x22, 0x4d, 0x44, 0x35, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x3a, 0x22, 0x4e, 0x33, 0x48, 0x65, 0x65, 0x51, 0x62, 0x48, 0x52, 0x5a, 0x62, 0x65, 0x53, 0x47, 0x35, 0x4c, 0x4c, 0x50, 0x39, 0x46, 0x2f, 0x41, 0x3d, 0x3d, 0x22, 0x7d, 0x80, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x65, 0x63, 0x64 }; private static readonly string FastRsyncLegacyDeltaExpectedFileHash = "N3HeeQbHRZbeSG5LLP9F/A=="; [Test] public void BinaryDeltaReader_ReadsLegacyDelta() { // Arrange var deltaStream = new MemoryStream(FastRsyncLegacyMetadataDelta); // Act IDeltaReader target = new BinaryDeltaReader(deltaStream, null); // Assert var hashAlgorithm = SupportedAlgorithms.Hashing.Default(); Assert.That(target.Type, Is.EqualTo(RsyncFormatType.FastRsync)); Assert.That(target.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name)); Assert.That(target.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes)); Assert.That(target.Metadata.ExpectedFileHash, Is.EqualTo(FastRsyncLegacyDeltaExpectedFileHash)); Assert.That(target.Metadata.ExpectedFileHashAlgorithm, Is.EqualTo("MD5")); Assert.That(target.Metadata.HashAlgorithm, Is.EqualTo(hashAlgorithm.Name)); Assert.That(target.Metadata.BaseFileHash, Is.Null); Assert.That(target.Metadata.BaseFileHashAlgorithm, Is.Null); } [Test] public void LegacyBinaryDeltaReader_ReadsDelta() { // Arrange var (_, baseSignatureStream, _, newDataStream) = Utils.PrepareTestData(16974, 8452, SignatureBuilder.DefaultChunkSize); var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); // Act var target = new BinaryDeltaReaderLegacy(deltaStream, null); // Assert var hashAlgorithm = SupportedAlgorithms.Hashing.Default(); Assert.That(target.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name)); Assert.That(target.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes)); Assert.That(target.Type, Is.EqualTo(RsyncFormatType.FastRsync)); Assert.That(target.Metadata.ExpectedFileHash, Is.Not.Null.And.Not.Empty); Assert.That(target.Metadata.ExpectedFileHashAlgorithm, Is.EqualTo("MD5")); Assert.That(target.Metadata.HashAlgorithm, Is.EqualTo(hashAlgorithm.Name)); } } ================================================ FILE: source/FastRsync.Tests/FastRsync.Tests.csproj ================================================  net10.0 PreserveNewest PreserveNewest PreserveNewest PreserveNewest PreserveNewest PreserveNewest PreserveNewest PreserveNewest ================================================ FILE: source/FastRsync.Tests/FastRsyncLegacy/BinaryDeltaReaderLegacy.cs ================================================ using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using Newtonsoft.Json; namespace FastRsync.Tests.FastRsyncLegacy; internal class BinaryFormat { public const int SignatureFormatHeaderLength = 7; // OCTOSIG or FRSNCSG public const int DeltaFormatHeaderLength = 9; // OCTODELTA or FRSNCDLTA public const byte CopyCommand = 0x60; public const byte DataCommand = 0x80; } internal class OctoBinaryFormat { public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG"); public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA"); public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>"); public const byte Version = 0x01; } internal class FastRsyncBinaryFormat { public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("FRSNCSG"); public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("FRSNCDLTA"); public const byte Version = 0x01; } public sealed class ProgressReport { public ProgressOperationType Operation { get; internal set; } public long CurrentPosition { get; internal set; } public long Total { get; internal set; } } internal class BinaryDeltaReaderLegacy : IDeltaReaderLegacy { private readonly BinaryReader reader; private readonly IProgress progressReport; private byte[] expectedHash; private IHashAlgorithm hashAlgorithm; private readonly int readBufferSize; public BinaryDeltaReaderLegacy(Stream stream, IProgress progressHandler, int readBufferSize = 4 * 1024 * 1024) { this.reader = new BinaryReader(stream); this.progressReport = progressHandler; this.readBufferSize = readBufferSize; } private DeltaMetadataLegacy _metadata; public DeltaMetadataLegacy Metadata { get { ReadMetadata(); return _metadata; } } public RsyncFormatType Type { get; private set; } public byte[] ExpectedHash { get { ReadMetadata(); return expectedHash; } } public IHashAlgorithm HashAlgorithm { get { ReadMetadata(); return hashAlgorithm; } } private void ReadMetadata() { if (_metadata != null) return; reader.BaseStream.Seek(0, SeekOrigin.Begin); var header = reader.ReadBytes(BinaryFormat.DeltaFormatHeaderLength); if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.DeltaHeader, header)) { ReadFastRsyncDeltaHeader(); return; } if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.DeltaHeader, header)) { ReadOctoDeltaHeader(); return; } throw new InvalidDataException("The delta file uses a different file format than this program can handle."); } private void ReadFastRsyncDeltaHeader() { var version = reader.ReadByte(); if (version != FastRsyncBinaryFormat.Version) throw new InvalidDataException("The delta file uses a newer file format than this program can handle."); var metadataStr = reader.ReadString(); _metadata = JsonConvert.DeserializeObject(metadataStr, NewtonsoftJsonSerializationSettings.JsonSettings); hashAlgorithm = SupportedAlgorithms.Hashing.Create(_metadata.HashAlgorithm); expectedHash = Convert.FromBase64String(_metadata.ExpectedFileHash); Type = RsyncFormatType.FastRsync; } private void ReadOctoDeltaHeader() { var version = reader.ReadByte(); if (version != OctoBinaryFormat.Version) throw new InvalidDataException("The delta file uses a newer file format than this program can handle."); var hashAlgorithmName = reader.ReadString(); hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName); var hashLength = reader.ReadInt32(); expectedHash = reader.ReadBytes(hashLength); var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length); if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta)) throw new InvalidDataException("The delta file appears to be corrupt."); _metadata = new DeltaMetadataLegacy { HashAlgorithm = hashAlgorithmName, ExpectedFileHashAlgorithm = hashAlgorithmName, ExpectedFileHash = Convert.ToBase64String(expectedHash) }; Type = RsyncFormatType.Octodiff; } public void Apply( Action writeData, Action copy) { var fileLength = reader.BaseStream.Length; ReadMetadata(); while (reader.BaseStream.Position != fileLength) { var b = reader.ReadByte(); progressReport?.Report(new ProgressReport { Operation = ProgressOperationType.ApplyingDelta, CurrentPosition = reader.BaseStream.Position, Total = fileLength }); if (b == BinaryFormat.CopyCommand) { var start = reader.ReadInt64(); var length = reader.ReadInt64(); copy(start, length); } else if (b == BinaryFormat.DataCommand) { var length = reader.ReadInt64(); long soFar = 0; while (soFar < length) { var bytes = reader.ReadBytes((int)Math.Min(length - soFar, readBufferSize)); soFar += bytes.Length; writeData(bytes); } } } } public async Task ApplyAsync( Func writeData, Func copy) { var fileLength = reader.BaseStream.Length; ReadMetadata(); var buffer = new byte[readBufferSize]; while (reader.BaseStream.Position != fileLength) { var b = reader.ReadByte(); progressReport?.Report(new ProgressReport { Operation = ProgressOperationType.ApplyingDelta, CurrentPosition = reader.BaseStream.Position, Total = fileLength }); if (b == BinaryFormat.CopyCommand) { var start = reader.ReadInt64(); var length = reader.ReadInt64(); await copy(start, length).ConfigureAwait(false); } else if (b == BinaryFormat.DataCommand) { var length = reader.ReadInt64(); long soFar = 0; while (soFar < length) { var bytesRead = await reader.BaseStream.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length)).ConfigureAwait(false); var bytes = buffer; if (bytesRead != buffer.Length) { bytes = new byte[bytesRead]; Array.Copy(buffer, bytes, bytesRead); } soFar += bytes.Length; await writeData(bytes).ConfigureAwait(false); } } } } } ================================================ FILE: source/FastRsync.Tests/FastRsyncLegacy/DeltaMetadataLegacy.cs ================================================ namespace FastRsync.Tests.FastRsyncLegacy; internal class DeltaMetadataLegacy { public string HashAlgorithm { get; set; } public string ExpectedFileHashAlgorithm { get; set; } public string ExpectedFileHash { get; set; } } ================================================ FILE: source/FastRsync.Tests/FastRsyncLegacy/IDeltaReaderLegacy.cs ================================================ using System; using System.Threading.Tasks; using FastRsync.Hash; using FastRsync.Signature; namespace FastRsync.Tests.FastRsyncLegacy; internal interface IDeltaReaderLegacy { byte[] ExpectedHash { get; } IHashAlgorithm HashAlgorithm { get; } DeltaMetadataLegacy Metadata { get; } RsyncFormatType Type { get; } void Apply( Action writeData, Action copy ); Task ApplyAsync( Func writeData, Func copy ); } ================================================ FILE: source/FastRsync.Tests/FastRsyncLegacy/NewtonsoftJsonSerializationSettings.cs ================================================ using Newtonsoft.Json.Serialization; using Newtonsoft.Json; namespace FastRsync.Tests.FastRsyncLegacy; public class NewtonsoftJsonSerializationSettings { static NewtonsoftJsonSerializationSettings() { JsonSettings = new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver(), Formatting = Formatting.None, NullValueHandling = NullValueHandling.Ignore }; } public static JsonSerializerSettings JsonSettings { get; } } ================================================ FILE: source/FastRsync.Tests/GZipTests.cs ================================================ using System; using System.IO; using System.IO.Compression; using System.Threading; using System.Threading.Tasks; using FastRsync.Compression; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Signature; using Newtonsoft.Json.Linq; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class GZipTests { [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] public void GZip_CompressRandomData(int dataLength) { // Arrange var data = new byte[dataLength]; // Main flow CompressData(data); } [Test] public void GZip_CompressData() { // Arrange var data = File.ReadAllBytes(".\\TestData\\basefile.bin"); // Main flow var destStream = CompressData(data); // Additional asserts Assert.That(destStream.Length, Is.EqualTo(6982738)); } private static MemoryStream CompressData(byte[] data) { // Arrange var srcStream = new MemoryStream(data); var destStream = new MemoryStream(); // Act GZip.Compress(srcStream, destStream); // Assert destStream.Seek(0, SeekOrigin.Begin); var decompressedStream = new MemoryStream(); using (var gz = new GZipStream(destStream, CompressionMode.Decompress, true)) { gz.CopyTo(decompressedStream); } var dataOutput = decompressedStream.ToArray(); Assert.That(dataOutput, Is.EqualTo(data)); return destStream; } [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] public void GZip_CompressRandomData_RsyncSignatureAndPatch(int dataLength) { // Arrange var dataBasis = new byte[dataLength]; new Random().NextBytes(dataBasis); var basisStream = new MemoryStream(dataBasis); var newFileStream = new MemoryStream(); newFileStream.Write(dataBasis, 10, dataLength * 4 / 5); var newRandomData = new byte[dataLength * 2 / 5]; new Random().NextBytes(newRandomData); newFileStream.Write(newRandomData, 0, newRandomData.Length); newFileStream.Seek(0, SeekOrigin.Begin); // Main flow FullCompressedRsyncFlow(basisStream, newFileStream); } [Test] public void GZip_CompressData_RsyncSignatureAndPatch() { // Arrange // Arrange var dataBasis = File.ReadAllBytes(".\\TestData\\basefile.bin"); var basisStream = new MemoryStream(dataBasis); var newData = File.ReadAllBytes(".\\TestData\\basefile2.bin"); var newFileStream = new MemoryStream(newData); // Main flow var deltaStream = FullCompressedRsyncFlow(basisStream, newFileStream); // Not calculated, taken from the valid flow Assert.That(deltaStream.Length, Is.EqualTo(662628)); } private static MemoryStream FullCompressedRsyncFlow(MemoryStream basisStream, MemoryStream newFileStream) { var basisStreamCompressed = new MemoryStream(); var basisStreamCompressedSignature = new MemoryStream(); var newFileStreamCompressed = new MemoryStream(); var deltaStream = new MemoryStream(); var patchedCompressedStream = new MemoryStream(); // Act GZip.Compress(basisStream, basisStreamCompressed); basisStreamCompressed.Seek(0, SeekOrigin.Begin); var signatureBuilder = new SignatureBuilder(); signatureBuilder.Build(basisStreamCompressed, new SignatureWriter(basisStreamCompressedSignature)); basisStreamCompressedSignature.Seek(0, SeekOrigin.Begin); GZip.Compress(newFileStream, newFileStreamCompressed); newFileStreamCompressed.Seek(0, SeekOrigin.Begin); var deltaBuilder = new DeltaBuilder(); deltaBuilder.BuildDelta(newFileStreamCompressed, new SignatureReader(basisStreamCompressedSignature, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); var deltaApplier = new DeltaApplier { SkipHashCheck = true }; var deltaReader = new BinaryDeltaReader(deltaStream, null); deltaApplier.Apply(basisStreamCompressed, deltaReader, patchedCompressedStream); deltaApplier.HashCheck(deltaReader, patchedCompressedStream); // Assert Assert.That(newFileStreamCompressed.ToArray(), Is.EqualTo(patchedCompressedStream.ToArray())); patchedCompressedStream.Seek(0, SeekOrigin.Begin); var decompressedStream = new MemoryStream(); using (var gz = new GZipStream(patchedCompressedStream, CompressionMode.Decompress)) { gz.CopyTo(decompressedStream); } var dataOutput = decompressedStream.ToArray(); Assert.That(dataOutput, Is.EqualTo(newFileStream.ToArray())); return deltaStream; } [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] public async Task GZip_CompressRandomDataAsync_RsyncSignatureAndPatch(int dataLength) { // Arrange var dataBasis = new byte[dataLength]; new Random().NextBytes(dataBasis); var newFileStream = new MemoryStream(); newFileStream.Write(dataBasis, 10, dataLength * 4 / 5); var newRandomData = new byte[dataLength * 2 / 5]; new Random().NextBytes(newRandomData); newFileStream.Write(newRandomData, 0, newRandomData.Length); newFileStream.Seek(0, SeekOrigin.Begin); var basisStream = new MemoryStream(dataBasis); // Main flow await FullCompressedRsyncFlowAsync(basisStream, newFileStream); } [Test] public async Task GZip_CompressDataAsync_RsyncSignatureAndPatch() { // Arrange var dataBasis = File.ReadAllBytes(".\\TestData\\basefile.bin"); var basisStream = new MemoryStream(dataBasis); var newData = File.ReadAllBytes(".\\TestData\\basefile2.bin"); var newFileStream = new MemoryStream(newData); // Main flow var deltaStream = await FullCompressedRsyncFlowAsync(basisStream, newFileStream); // Not calculated, taken from the valid flow Assert.That(deltaStream.Length, Is.EqualTo(662628)); } private static async Task FullCompressedRsyncFlowAsync(MemoryStream basisStream, MemoryStream newFileStream) { // Arrange var basisStreamCompressed = new MemoryStream(); var basisStreamCompressedSignature = new MemoryStream(); var newFileStreamCompressed = new MemoryStream(); var deltaStream = new MemoryStream(); var patchedCompressedStream = new MemoryStream(); // Act await GZip.CompressAsync(basisStream, basisStreamCompressed); basisStreamCompressed.Seek(0, SeekOrigin.Begin); var signatureBuilder = new SignatureBuilder(); await signatureBuilder.BuildAsync(basisStreamCompressed, new SignatureWriter(basisStreamCompressedSignature)); basisStreamCompressedSignature.Seek(0, SeekOrigin.Begin); await GZip.CompressAsync(newFileStream, newFileStreamCompressed); newFileStreamCompressed.Seek(0, SeekOrigin.Begin); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newFileStreamCompressed, new SignatureReader(basisStreamCompressedSignature, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); var deltaApplier = new DeltaApplier { SkipHashCheck = true }; var deltaReader = new BinaryDeltaReader(deltaStream, null); await deltaApplier.ApplyAsync(basisStreamCompressed, deltaReader, patchedCompressedStream); await deltaApplier.HashCheckAsync(deltaReader, patchedCompressedStream); // Assert Assert.That(newFileStreamCompressed.ToArray(), Is.EqualTo(patchedCompressedStream.ToArray())); patchedCompressedStream.Seek(0, SeekOrigin.Begin); var decompressedStream = new MemoryStream(); await using (var gz = new GZipStream(patchedCompressedStream, CompressionMode.Decompress)) { await gz.CopyToAsync(decompressedStream); } Assert.That(newFileStream.ToArray(), Is.EqualTo(decompressedStream.ToArray())); return deltaStream; } } ================================================ FILE: source/FastRsync.Tests/HashTests.cs ================================================ using NUnit.Framework; using System; using System.IO; using System.Threading.Tasks; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; using System.Data.HashFunction.xxHash; using System.IO.Hashing; namespace FastRsync.Tests; [TestFixture] public class HashTests { [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] [TestCase(6 * 1024 * 1024)] [TestCase(7 * 1023 * 1017)] [TestCase(100 * 333 * 333)] public void XxHash64StaticBackwardCompatibility(int dataLength) { // Arrange var data = new byte[dataLength]; new Random().NextBytes(data); // Act var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig { HashSizeInBits = 64 }); var dataXxHash64Dest = dataXxHash.ComputeHash(data).Hash; var systemXxHash64Dest = XxHash64.Hash(data); Array.Reverse(systemXxHash64Dest); // Assert Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest)); } [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] [TestCase(6 * 1024 * 1024)] [TestCase(7 * 1023 * 1017)] [TestCase(100 * 333 * 333)] public void XxHash64BackwardCompatibility(int dataLength) { // Arrange var data = new byte[dataLength]; new Random().NextBytes(data); // Act var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig { HashSizeInBits = 64 }); NonCryptographicHashAlgorithm systemXxHash = new XxHash64(); var dataXxHash64Dest = dataXxHash.ComputeHash(data).Hash; systemXxHash.Append(data); var systemXxHash64Dest = systemXxHash.GetHashAndReset(); Array.Reverse(systemXxHash64Dest); // Assert Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest)); } [Test] [TestCase(120)] [TestCase(5 * 1024 * 1024)] [TestCase(100 * 333 * 333)] public async Task XxHash64StreamBackwardCompatibility(int dataLength) { // Arrange var data = new byte[dataLength]; new Random().NextBytes(data); var streamdata = new MemoryStream(data); const string azureBlobStorageConnectionString = "UseDevelopmentStorage=true;"; var azureBlobServiceClient = new BlobServiceClient(azureBlobStorageConnectionString); var azureBlobContainerClient = azureBlobServiceClient.GetBlobContainerClient("hashbenchmark"); await azureBlobContainerClient.CreateIfNotExistsAsync(); var azureBlob = azureBlobContainerClient.GetBlockBlobClient("XxHash64StreamBackwardCompatibility_" + dataLength); await azureBlob.UploadAsync(streamdata); await using var blobstreamDataXxHash = await azureBlob.OpenReadAsync(); await using var blobstreamSystemXxHash = await azureBlob.OpenReadAsync(); // Act var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig { HashSizeInBits = 64 }); NonCryptographicHashAlgorithm systemXxHash = new XxHash64(); var dataXxHash64Dest = (await dataXxHash.ComputeHashAsync(blobstreamDataXxHash)).Hash; await systemXxHash.AppendAsync(blobstreamSystemXxHash); var systemXxHash64Dest = systemXxHash.GetHashAndReset(); Array.Reverse(systemXxHash64Dest); // Assert Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest)); } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffDeltaWriter.cs ================================================ using System.IO; using System.Threading.Tasks; using FastRsync.Delta; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; public interface IOctodiffDeltaWriter { void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash); void WriteCopyCommand(DataRange segment); void WriteDataCommand(Stream source, long offset, long length); void Finish(); } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffSignatureWriter.cs ================================================ using System.IO; using FastRsync.Core; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; public interface IOctodiffSignatureWriter { void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm); void WriteChunk(ChunkSignature signature); } public class OctodiffSignatureWriter(Stream signatureStream) : IOctodiffSignatureWriter { private readonly BinaryWriter signaturebw = new(signatureStream); public void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) { signaturebw.Write(OctodiffBinaryFormat.SignatureHeader); signaturebw.Write(OctodiffBinaryFormat.Version); signaturebw.Write(hashAlgorithm.Name); signaturebw.Write(rollingChecksumAlgorithm.Name); signaturebw.Write(OctodiffBinaryFormat.EndOfMetadata); } public void WriteChunk(ChunkSignature signature) { signaturebw.Write(signature.Length); signaturebw.Write(signature.RollingChecksum); signaturebw.Write(signature.Hash); } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffAggregateCopyOperationsDecorator.cs ================================================ using System.IO; using FastRsync.Delta; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; // This decorator turns any sequential copy operations into a single operation, reducing // the size of the delta file. // For example: // Copy: 0x0000 - 0x0400 // Copy: 0x0401 - 0x0800 // Copy: 0x0801 - 0x0C00 // Gets turned into: // Copy: 0x0000 - 0x0C00 public class OctodiffAggregateCopyOperationsDecorator : IOctodiffDeltaWriter { private readonly IOctodiffDeltaWriter decorated; private DataRange bufferedCopy; public OctodiffAggregateCopyOperationsDecorator(IOctodiffDeltaWriter decorated) { this.decorated = decorated; } public void WriteDataCommand(Stream source, long offset, long length) { FlushCurrentCopyCommand(); decorated.WriteDataCommand(source, offset, length); } public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash) { decorated.WriteMetadata(hashAlgorithm, expectedNewFileHash); } public void WriteCopyCommand(DataRange chunk) { if (bufferedCopy.Length > 0 && bufferedCopy.StartOffset + bufferedCopy.Length == chunk.StartOffset) { bufferedCopy.Length += chunk.Length; } else { FlushCurrentCopyCommand(); bufferedCopy = chunk; } } private void FlushCurrentCopyCommand() { if (bufferedCopy.Length <= 0) { return; } decorated.WriteCopyCommand(bufferedCopy); bufferedCopy = new DataRange(); } public void Finish() { FlushCurrentCopyCommand(); decorated.Finish(); } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryDeltaWriter.cs ================================================ using System; using System.IO; using FastRsync.Delta; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; public class OctodiffBinaryDeltaWriter : IOctodiffDeltaWriter { private readonly BinaryWriter writer; private readonly int readWriteBufferSize; public OctodiffBinaryDeltaWriter(Stream stream, int readWriteBufferSize = 1024 * 1024) { writer = new BinaryWriter(stream); this.readWriteBufferSize = readWriteBufferSize; } public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash) { writer.Write(OctodiffBinaryFormat.DeltaHeader); writer.Write(OctodiffBinaryFormat.Version); writer.Write(hashAlgorithm.Name); writer.Write(expectedNewFileHash.Length); writer.Write(expectedNewFileHash); writer.Write(OctodiffBinaryFormat.EndOfMetadata); } public void WriteCopyCommand(DataRange segment) { writer.Write(OctodiffBinaryFormat.CopyCommand); writer.Write(segment.StartOffset); writer.Write(segment.Length); } public void WriteDataCommand(Stream source, long offset, long length) { writer.Write(OctodiffBinaryFormat.DataCommand); writer.Write(length); var originalPosition = source.Position; try { source.Seek(offset, SeekOrigin.Begin); var buffer = new byte[Math.Min((int)length, readWriteBufferSize)]; int read; long soFar = 0; while ((read = source.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0) { soFar += read; writer.Write(buffer, 0, read); } } finally { source.Seek(originalPosition, SeekOrigin.Begin); } } public void Finish() { } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryFormat.cs ================================================ using System.Text; namespace FastRsync.Tests.OctodiffLegacy; class OctodiffBinaryFormat { public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG"); public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA"); public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>"); public const byte CopyCommand = 0x60; public const byte DataCommand = 0x80; public const byte Version = 0x01; } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffDeltaBuilder.cs ================================================ using System.Collections; using System.Collections.Generic; using System.IO; using FastRsync.Core; using FastRsync.Delta; namespace FastRsync.Tests.OctodiffLegacy; public class OctodiffDeltaBuilder { private readonly int readBufferSize; public OctodiffDeltaBuilder(int readBufferSize = 4 * 1024 * 1024) { this.readBufferSize = readBufferSize; } public void BuildDelta(Stream newFileStream, IOctodiffSignatureReader signatureReader, IOctodiffDeltaWriter deltaWriter) { var signature = signatureReader.ReadSignature(); var chunks = signature.Chunks; var newFileHash = signature.HashAlgorithm.ComputeHash(newFileStream); newFileStream.Seek(0, SeekOrigin.Begin); deltaWriter.WriteMetadata(signature.HashAlgorithm, newFileHash); chunks = OrderChunksByChecksum(chunks); var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize); var buffer = new byte[readBufferSize]; long lastMatchPosition = 0; while (true) { var startPosition = newFileStream.Position; var read = newFileStream.Read(buffer, 0, buffer.Length); if (read <= 0) break; var checksumAlgorithm = signature.RollingChecksumAlgorithm; uint checksum = 0; var remainingPossibleChunkSize = maxChunkSize; for (var i = 0; i < read - minChunkSize + 1; i++) { var readSoFar = startPosition + i; var remainingBytes = read - i; if (remainingBytes < maxChunkSize) { remainingPossibleChunkSize = minChunkSize; } if (i == 0 || remainingBytes < maxChunkSize) { checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize); } else { var remove = buffer[i - 1]; var add = buffer[i + remainingPossibleChunkSize - 1]; checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize); } if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize) continue; if (!chunkMap.ContainsKey(checksum)) continue; var startIndex = chunkMap[checksum]; for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++) { var chunk = chunks[j]; var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize); if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash)) { readSoFar = readSoFar + remainingPossibleChunkSize; var missing = readSoFar - lastMatchPosition; if (missing > remainingPossibleChunkSize) { deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize); } deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length)); lastMatchPosition = readSoFar; break; } } } if (read < buffer.Length) { break; } newFileStream.Position = newFileStream.Position - maxChunkSize + 1; } if (newFileStream.Length != lastMatchPosition) { deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition); } deltaWriter.Finish(); } private class ChunkSignatureChecksumComparer : IComparer { public int Compare(ChunkSignature x, ChunkSignature y) { var comparison = x.RollingChecksum.CompareTo(y.RollingChecksum); return comparison == 0 ? x.StartOffset.CompareTo(y.StartOffset) : comparison; } } private static List OrderChunksByChecksum(List chunks) { chunks.Sort(new ChunkSignatureChecksumComparer()); return chunks; } private Dictionary CreateChunkMap(IList chunks, out int maxChunkSize, out int minChunkSize) { maxChunkSize = 0; minChunkSize = int.MaxValue; var chunkMap = new Dictionary(); for (var i = 0; i < chunks.Count; i++) { var chunk = chunks[i]; if (chunk.Length > maxChunkSize) { maxChunkSize = chunk.Length; } if (chunk.Length < minChunkSize) { minChunkSize = chunk.Length; } if (!chunkMap.ContainsKey(chunk.RollingChecksum)) { chunkMap[chunk.RollingChecksum] = i; } } return chunkMap; } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignature.cs ================================================ using System.Collections.Generic; using FastRsync.Core; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; public class OctodiffSignature { public OctodiffSignature(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) { HashAlgorithm = hashAlgorithm; RollingChecksumAlgorithm = rollingChecksumAlgorithm; Chunks = new List(); } public IHashAlgorithm HashAlgorithm { get; private set; } public IRollingChecksum RollingChecksumAlgorithm { get; private set; } public List Chunks { get; private set; } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureBuilder.cs ================================================ using System; using System.IO; using FastRsync.Core; using FastRsync.Hash; namespace FastRsync.Tests.OctodiffLegacy; public class OctodiffSignatureBuilder { public const short MinimumChunkSize = 128; public const short DefaultChunkSize = 2048; public const short MaximumChunkSize = 31 * 1024; private short chunkSize; public OctodiffSignatureBuilder() : this(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Default()) { } public OctodiffSignatureBuilder(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) { HashAlgorithm = hashAlgorithm; RollingChecksumAlgorithm = rollingChecksumAlgorithm; ChunkSize = DefaultChunkSize; } public IHashAlgorithm HashAlgorithm { get; set; } public IRollingChecksum RollingChecksumAlgorithm { get; set; } public short ChunkSize { get => chunkSize; set { if (value < MinimumChunkSize) throw new ArgumentException($"Chunk size cannot be less than {MinimumChunkSize}"); if (value > MaximumChunkSize) throw new ArgumentException($"Chunk size cannot be exceed {MaximumChunkSize}"); chunkSize = value; } } public void Build(Stream stream, IOctodiffSignatureWriter signatureWriter) { WriteMetadata(stream, signatureWriter); WriteChunkSignatures(stream, signatureWriter); } private void WriteMetadata(Stream stream, IOctodiffSignatureWriter signatureWriter) { stream.Seek(0, SeekOrigin.Begin); signatureWriter.WriteMetadata(HashAlgorithm, RollingChecksumAlgorithm); } private void WriteChunkSignatures(Stream stream, IOctodiffSignatureWriter signatureWriter) { var checksumAlgorithm = RollingChecksumAlgorithm; var hashAlgorithm = HashAlgorithm; stream.Seek(0, SeekOrigin.Begin); long start = 0; int read; var block = new byte[ChunkSize]; while ((read = stream.Read(block, 0, block.Length)) > 0) { signatureWriter.WriteChunk(new ChunkSignature { StartOffset = start, Length = (short)read, Hash = hashAlgorithm.ComputeHash(block, 0, read), RollingChecksum = checksumAlgorithm.Calculate(block, 0, read) }); start += read; } } } ================================================ FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureReader.cs ================================================ using System; using System.Collections; using System.IO; using FastRsync.Core; using FastRsync.Diagnostics; namespace FastRsync.Tests.OctodiffLegacy; public interface IOctodiffSignatureReader { OctodiffSignature ReadSignature(); } public class OctodiffSignatureReader : IOctodiffSignatureReader { private readonly BinaryReader reader; public OctodiffSignatureReader(Stream stream, IProgress progressHandler) { this.reader = new BinaryReader(stream); } public OctodiffSignature ReadSignature() { var header = reader.ReadBytes(OctodiffBinaryFormat.SignatureHeader.Length); if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctodiffBinaryFormat.SignatureHeader, header)) throw new InvalidDataException("The signature file appears to be corrupt."); var version = reader.ReadByte(); if (version != OctodiffBinaryFormat.Version) throw new InvalidDataException("The signature file uses a newer file format than this program can handle."); var hashAlgorithm = reader.ReadString(); var rollingChecksumAlgorithm = reader.ReadString(); var endOfMeta = reader.ReadBytes(OctodiffBinaryFormat.EndOfMetadata.Length); if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctodiffBinaryFormat.EndOfMetadata, endOfMeta)) throw new InvalidDataException("The signature file appears to be corrupt."); var hashAlgo = SupportedAlgorithms.Hashing.Create(hashAlgorithm); var signature = new OctodiffSignature( hashAlgo, SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm)); var expectedHashLength = hashAlgo.HashLengthInBytes; long start = 0; var fileLength = reader.BaseStream.Length; var remainingBytes = fileLength - reader.BaseStream.Position; var signatureSize = sizeof(ushort) + sizeof(uint) + expectedHashLength; if (remainingBytes % signatureSize != 0) throw new InvalidDataException("The signature file appears to be corrupt; at least one chunk has data missing."); while (reader.BaseStream.Position < fileLength - 1) { var length = reader.ReadInt16(); var checksum = reader.ReadUInt32(); var chunkHash = reader.ReadBytes(expectedHashLength); signature.Chunks.Add(new ChunkSignature { StartOffset = start, Length = length, RollingChecksum = checksum, Hash = chunkHash }); start += length; } return signature; } } ================================================ FILE: source/FastRsync.Tests/PatchingAsyncTests.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using FastRsync.Tests.OctodiffLegacy; using NSubstitute; using NUnit.Framework; using NUnit.Framework.Legacy; namespace FastRsync.Tests; [TestFixture] public class PatchingAsyncTests { [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")] public async Task PatchingAsyncXXHash_ForNewData_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize, string signatureHashingAlgorithm, string rollingChecksumAlgorithm) { // Arrange var signatureHash = SupportedAlgorithms.Hashing.Create(signatureHashingAlgorithm); var rollingChecksum = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm); var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, chunkSize, rollingChecksum, signatureHash).ConfigureAwait(false); var progressReporter = Substitute.For>(); // Act var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)] public async Task PatchingAsyncXXHash_ForOctodiffSignature_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { // Arrange var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize); var progressReporter = Substitute.For>(); // Act var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)] public async Task PatchingAsyncXXHash_ForOctodiffSignatureAndPatch_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { // Arrange var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize); var progressReporter = Substitute.For>(); var deltaStream = new MemoryStream(); var deltaBuilder = new OctodiffDeltaBuilder(); deltaBuilder.BuildDelta(newDataStream, new OctodiffSignatureReader(baseSignatureStream, null), new OctodiffAggregateCopyOperationsDecorator(new OctodiffBinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); // Act var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(8467, SignatureBuilder.DefaultChunkSize)] public async Task PatchingAsyncXXHash_ForTheSameData_PatchesFile(int numberOfBytes, short chunkSize) { // Arrange var baseData = new byte[numberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new SignatureBuilder { ChunkSize = chunkSize }; signatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureStream)); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newDataStream = new MemoryStream(baseData); var progressReporter = Substitute.For>(); // Act var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false); deltaStream.Seek(0, SeekOrigin.Begin); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(baseData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } private static async Task<(MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream)> PrepareTestDataAsync(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize, Hash.IRollingChecksum rollingChecksumAlg, IHashAlgorithm signatureHashingAlg) { var baseData = new byte[baseNumberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new SignatureBuilder(signatureHashingAlg, rollingChecksumAlg) { ChunkSize = chunkSize }; await signatureBuilder.BuildAsync(baseDataStream, new SignatureWriter(baseSignatureStream)).ConfigureAwait(false); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newData = new byte[newDataNumberOfBytes]; new Random().NextBytes(newData); var newDataStream = new MemoryStream(newData); return (baseDataStream, baseSignatureStream, newData, newDataStream); } private static (MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream) PrepareTestDataWithOctodiffSignature(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { var baseData = new byte[baseNumberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new OctodiffSignatureBuilder { ChunkSize = chunkSize }; signatureBuilder.Build(baseDataStream, new OctodiffSignatureWriter(baseSignatureStream)); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newData = new byte[newDataNumberOfBytes]; new Random().NextBytes(newData); var newDataStream = new MemoryStream(newData); return (baseDataStream, baseSignatureStream, newData, newDataStream); } } ================================================ FILE: source/FastRsync.Tests/PatchingBigFilesTests.cs ================================================ using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; using System; using System.IO; using System.Security.Cryptography; using FastRsync.Hash; namespace FastRsync.Tests; [TestFixture] public class PatchingBigFilesTests { [Test] [TestCase(2050, 2790, "XXH64", "Adler32")] [TestCase(2050, 2790, "XXH3", "Adler32")] [TestCase(2050, 2790, "XXH64", "Adler32V3")] [TestCase(2050, 2790, "XXH3", "Adler32V3")] public void PatchingSyncXXHash_BigFile(int baseNumberOfMBytes, int newNumberOfMBytes, string signatureHashingAlgorithm, string rollingChecksumAlgorithm) { // Arrange var signatureHash = SupportedAlgorithms.Hashing.Create(signatureHashingAlgorithm); var rollingChecksum = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm); var baseFileName = Path.GetTempFileName(); var newFileName = Path.GetTempFileName(); try { var deltaFileName = Path.GetTempFileName(); var patchedFileName = Path.GetTempFileName(); var progressReporter = Substitute.For>(); { var buffer = new byte[16384]; var rng = new Random(); using var baseFileStream = new FileStream(baseFileName, FileMode.Create); for (var i = 0; i < baseNumberOfMBytes * 64; i++) { rng.NextBytes(buffer); baseFileStream.Write(buffer, 0, buffer.Length); } baseFileStream.Seek(0, SeekOrigin.Begin); using var newFileStream = new FileStream(newFileName, FileMode.Create); for (var i = 0; i < baseNumberOfMBytes * 64; i++) { rng.NextBytes(buffer); newFileStream.Write(buffer, 0, buffer.Length); } newFileStream.Seek(0, SeekOrigin.Begin); using var baseSignatureStream = PrepareTestData(baseFileStream, signatureHash, rollingChecksum); // Act using var deltaStream = new FileStream(deltaFileName, FileMode.OpenOrCreate); using var patchedDataStream = new FileStream(patchedFileName, FileMode.OpenOrCreate); var deltaBuilder = new DeltaBuilder(); deltaBuilder.BuildDelta(newFileStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); var deltaApplier = new DeltaApplier(); deltaApplier.Apply(baseFileStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream); } // Assert Assert.That(new FileInfo(newFileName).Length, Is.EqualTo(new FileInfo(patchedFileName).Length)); Assert.That(CompareFilesByHash(newFileName, patchedFileName), Is.True); progressReporter.Received().Report(Arg.Any()); } catch (Exception) { Assert.Fail(); } finally { File.Delete(baseFileName); File.Delete(newFileName); } } public static Stream PrepareTestData(Stream baseDataStream, IHashAlgorithm signatureHashingAlgorithm, IRollingChecksum rollingChecksumAlgorithm) { var baseSignatureStream = new MemoryStream(); var signatureBuilder = new SignatureBuilder(signatureHashingAlgorithm, rollingChecksumAlgorithm); signatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureStream)); baseSignatureStream.Seek(0, SeekOrigin.Begin); return baseSignatureStream; } public static bool CompareFilesByHash(string fileName1, string fileName2) { byte[] hash1, hash2; using (var stream = File.OpenRead(fileName1)) hash1 = MD5.Create().ComputeHash(stream); using (var stream = File.OpenRead(fileName2)) hash2 = MD5.Create().ComputeHash(stream); return hash1.SequenceEqual(hash2); } } ================================================ FILE: source/FastRsync.Tests/PatchingSyncTests.cs ================================================ using System; using System.IO; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using FastRsync.Tests.OctodiffLegacy; using NSubstitute; using NUnit.Framework; using NUnit.Framework.Legacy; namespace FastRsync.Tests; [TestFixture] public class PatchingSyncTests { [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)] public void PatchingSyncXXHash_ForNewData_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { // Arrange var (baseDataStream, baseSignatureStream, newData, newDataStream) = Utils.PrepareTestData(baseNumberOfBytes, newDataNumberOfBytes, chunkSize); var progressReporter = Substitute.For>(); // Act var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); deltaApplier.Apply(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)] public void PatchingSyncXXHash_ForOctodiffSignature_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { // Arrange var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize); var progressReporter = Substitute.For>(); // Act var deltaStream = new MemoryStream(); var deltaBuilder = new DeltaBuilder(); deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); deltaApplier.Apply(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)] [TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)] [TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)] [TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)] [TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)] public void PatchingSyncXXHash_ForOctodiffSignatureAndPatch_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { // Arrange var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize); var progressReporter = Substitute.For>(); var deltaStream = new MemoryStream(); var deltaBuilder = new OctodiffDeltaBuilder(); deltaBuilder.BuildDelta(newDataStream, new OctodiffSignatureReader(baseSignatureStream, null), new OctodiffAggregateCopyOperationsDecorator(new OctodiffBinaryDeltaWriter(deltaStream))); deltaStream.Seek(0, SeekOrigin.Begin); // Act var patchedDataStream = new MemoryStream(); var deltaApplier = new DeltaApplier(); deltaApplier.Apply(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream); // Assert CollectionAssert.AreEqual(newData, patchedDataStream.ToArray()); progressReporter.Received().Report(Arg.Any()); } private static (MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream) PrepareTestDataWithOctodiffSignature(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { var baseData = new byte[baseNumberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new OctodiffSignatureBuilder { ChunkSize = chunkSize }; signatureBuilder.Build(baseDataStream, new OctodiffSignatureWriter(baseSignatureStream)); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newData = new byte[newDataNumberOfBytes]; new Random().NextBytes(newData); var newDataStream = new MemoryStream(newData); return (baseDataStream, baseSignatureStream, newData, newDataStream); } } ================================================ FILE: source/FastRsync.Tests/SignatureBuilderAsyncRandomDataTests.cs ================================================ using System; using System.IO; using System.Security.Cryptography; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class SignatureBuilderAsyncRandomDataTests { [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public async Task SignatureBuilderAsyncXXHash_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ChunkSize = chunkSize, ProgressReport = progressReporter }; await target.BuildAsync(dataStream, new SignatureWriter(signatureStream)).ConfigureAwait(false); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public async Task SignatureBuilderAsyncXXHashAdlerV3_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32RollingV3()) { ChunkSize = chunkSize, ProgressReport = progressReporter }; await target.BuildAsync(dataStream, new SignatureWriter(signatureStream)).ConfigureAwait(false); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksumV3()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public async Task SignatureBuilderAsyncSha1_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Adler32Rolling()) { ChunkSize = chunkSize, ProgressReport = progressReporter }; await target.BuildAsync(dataStream, new SignatureWriter(signatureStream)).ConfigureAwait(false); // Assert CommonAsserts.ValidateSignature(signatureStream, new CryptographyHashAlgorithmWrapper("SHA1", SHA1.Create()), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } } ================================================ FILE: source/FastRsync.Tests/SignatureBuilderSyncRandomDataTests.cs ================================================ using System; using System.IO; using System.Security.Cryptography; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class SignatureBuilderSyncRandomDataTests { [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public void SignatureBuilderXXHash_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ChunkSize = chunkSize, ProgressReport = progressReporter }; target.Build(dataStream, new SignatureWriter(signatureStream)); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public void SignatureBuilderXXHashAdlerV3_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32RollingV3()) { ChunkSize = chunkSize, ProgressReport = progressReporter }; target.Build(dataStream, new SignatureWriter(signatureStream)); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksumV3()); progressReporter.Received().Report(Arg.Any()); } [Test] [TestCase(2, SignatureBuilder.MinimumChunkSize)] [TestCase(10, SignatureBuilder.MinimumChunkSize)] [TestCase(16974, SignatureBuilder.MinimumChunkSize)] [TestCase(2, SignatureBuilder.DefaultChunkSize)] [TestCase(10, SignatureBuilder.DefaultChunkSize)] [TestCase(16974, SignatureBuilder.DefaultChunkSize)] [TestCase(2, SignatureBuilder.MaximumChunkSize)] [TestCase(10, SignatureBuilder.MaximumChunkSize)] [TestCase(16974, SignatureBuilder.MaximumChunkSize)] public void SignatureBuilderSha1_ForRandomData_BuildsSignature(int numberOfBytes, short chunkSize) { // Arrange var data = new byte[numberOfBytes]; new Random().NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Adler32Rolling()) { ChunkSize = chunkSize, ProgressReport = progressReporter }; target.Build(dataStream, new SignatureWriter(signatureStream)); // Assert CommonAsserts.ValidateSignature(signatureStream, new CryptographyHashAlgorithmWrapper("SHA1", SHA1.Create()), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } } ================================================ FILE: source/FastRsync.Tests/SignatureBuilderTests.cs ================================================ using System; using System.IO; using System.Threading.Tasks; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; using NUnit.Framework.Legacy; namespace FastRsync.Tests; [TestFixture] public class SignatureBuilderTests { private const int RandomSeed = 123; private readonly byte[] xxhash1037TestSignature = { 0x46, 0x52, 0x53, 0x4E, 0x43, 0x53, 0x47, 0x01, 0x8B, 0x01, 0x7B, 0x22, 0x63, 0x68, 0x75, 0x6E, 0x6B, 0x48, 0x61, 0x73, 0x68, 0x41, 0x6C, 0x67, 0x6F, 0x72, 0x69, 0x74, 0x68, 0x6D, 0x22, 0x3A, 0x22, 0x58, 0x58, 0x48, 0x36, 0x34, 0x22, 0x2C, 0x22, 0x72, 0x6F, 0x6C, 0x6C, 0x69, 0x6E, 0x67, 0x43, 0x68, 0x65, 0x63, 0x6B, 0x73, 0x75, 0x6D, 0x41, 0x6C, 0x67, 0x6F, 0x72, 0x69, 0x74, 0x68, 0x6D, 0x22, 0x3A, 0x22, 0x41, 0x64, 0x6C, 0x65, 0x72, 0x33, 0x32, 0x22, 0x2C, 0x22, 0x62, 0x61, 0x73, 0x65, 0x46, 0x69, 0x6C, 0x65, 0x48, 0x61, 0x73, 0x68, 0x41, 0x6C, 0x67, 0x6F, 0x72, 0x69, 0x74, 0x68, 0x6D, 0x22, 0x3A, 0x22, 0x4D, 0x44, 0x35, 0x22, 0x2C, 0x22, 0x62, 0x61, 0x73, 0x65, 0x46, 0x69, 0x6C, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x3A, 0x22, 0x41, 0x33, 0x37, 0x45, 0x79, 0x65, 0x6A, 0x4E, 0x6E, 0x4B, 0x6F, 0x6C, 0x62, 0x68, 0x64, 0x34, 0x68, 0x73, 0x6F, 0x4E, 0x6F, 0x51, 0x3D, 0x3D, 0x22, 0x7D, 0x0D, 0x04, 0x2F, 0xFC, 0xF4, 0x6C, 0x7B, 0x52, 0x06, 0x17, 0x0A, 0x90, 0x3D, 0x70 }; [Test] public void SignatureBuilderXXHash_BuildsSignature() { // Arrange const int dataLength = 1037; var data = new byte[dataLength]; new Random(RandomSeed).NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ProgressReport = progressReporter }; target.Build(dataStream, new SignatureWriter(signatureStream)); // Assert CollectionAssert.AreEqual(xxhash1037TestSignature, signatureStream.ToArray()); CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } [Test] public async Task SignatureBuilderAsyncXXHash_BuildsSignature() { // Arrange const int dataLength = 1037; var data = new byte[dataLength]; new Random(RandomSeed).NextBytes(data); var dataStream = new MemoryStream(data); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ProgressReport = progressReporter }; await target.BuildAsync(dataStream, new SignatureWriter(signatureStream)).ConfigureAwait(false); // Assert CollectionAssert.AreEqual(xxhash1037TestSignature, signatureStream.ToArray()); CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(data), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } [Test] public async Task SignatureBuilderAsyncXXHash_ForEmptyStream_BuildsSignature() { // Arrange var dataStream = new MemoryStream(); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ProgressReport = progressReporter }; await target.BuildAsync(dataStream, new SignatureWriter(signatureStream)).ConfigureAwait(false); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(dataStream.ToArray()), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } [Test] public void SignatureBuilderSyncXXHash_ForEmptyStream_BuildsSignature() { // Arrange var dataStream = new MemoryStream(); var signatureStream = new MemoryStream(); var progressReporter = Substitute.For>(); // Act var target = new SignatureBuilder { ProgressReport = progressReporter }; target.Build(dataStream, new SignatureWriter(signatureStream)); // Assert CommonAsserts.ValidateSignature(signatureStream, SupportedAlgorithms.Hashing.XxHash(), Utils.GetMd5(dataStream.ToArray()), new Adler32RollingChecksum()); progressReporter.Received().Report(Arg.Any()); } } ================================================ FILE: source/FastRsync.Tests/SignatureReaderTests.cs ================================================ using System; using System.IO; using FastRsync.Core; using FastRsync.Diagnostics; using FastRsync.Hash; using FastRsync.Signature; using NSubstitute; using NUnit.Framework; namespace FastRsync.Tests; [TestFixture] public class SignatureReaderTests { [Test] public void SignatureReader_ReadsOctodiffSignature() { // Arrange byte[] octodiffxxHashSignature = { 0x4F, 0x43, 0x54, 0x4F, 0x53, 0x49, 0x47, 0x01, 0x05, 0x58, 0x58, 0x48, 0x36, 0x34, 0x07, 0x41, 0x64, 0x6C, 0x65, 0x72, 0x33, 0x32, 0x3E, 0x3E, 0x3E, 0x0D, 0x04, 0x2F, 0xFC, 0xF4, 0x6C, 0x7B, 0x52, 0x06, 0x17, 0x0A, 0x90, 0x3D, 0x70 }; var signatureStream = new MemoryStream(octodiffxxHashSignature); var progressReporter = Substitute.For>(); // Act var target = new SignatureReader(signatureStream, progressReporter).ReadSignature(); // Assert var hashAlgorithm = SupportedAlgorithms.Hashing.Default(); Assert.That(target.Type, Is.EqualTo(RsyncFormatType.Octodiff) ); Assert.That(target.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name)); Assert.That(target.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes)); Assert.That(target.RollingChecksumAlgorithm.Name, Is.EqualTo(new Adler32RollingChecksum().Name)); progressReporter.Received().Report(Arg.Any()); } [Test] public void SignatureReader_ReadsRandomData_ThrowsException() { // Arrange var signatureData = new byte[1458]; new Random().NextBytes(signatureData); var signatureStream = new MemoryStream(signatureData); var progressReporter = Substitute.For>(); // Act var target = new SignatureReader(signatureStream, progressReporter); // Assert Assert.Throws(() => target.ReadSignature()); } } ================================================ FILE: source/FastRsync.Tests/Utils.cs ================================================ using System; using System.IO; using System.Security.Cryptography; using FastRsync.Signature; namespace FastRsync.Tests; class Utils { public static string GetMd5(byte[] data) { using MD5 md5Hash = MD5.Create(); return Convert.ToBase64String(md5Hash.ComputeHash(data)); } public static (MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream) PrepareTestData(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize) { var baseData = new byte[baseNumberOfBytes]; new Random().NextBytes(baseData); var baseDataStream = new MemoryStream(baseData); var baseSignatureStream = new MemoryStream(); var signatureBuilder = new SignatureBuilder { ChunkSize = chunkSize }; signatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureStream)); baseSignatureStream.Seek(0, SeekOrigin.Begin); var newData = new byte[newDataNumberOfBytes]; new Random().NextBytes(newData); var newDataStream = new MemoryStream(newData); return (baseDataStream, baseSignatureStream, newData, newDataStream); } } ================================================ FILE: source/FastRsync.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34309.116 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octodiff", "Octodiff\Octodiff.csproj", "{BBA7CDC2-DE25-4131-89F2-506712178FC8}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octodiff.Tests", "Octodiff.Tests\Octodiff.Tests.csproj", "{B7E95BD1-676E-4AC7-9089-3270392776BA}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastRsync", "FastRsync\FastRsync.csproj", "{7B6474F5-8EE1-4D09-A48A-5A295E2DF8E2}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastRsync.Tests", "FastRsync.Tests\FastRsync.Tests.csproj", "{CF4B2B1E-71C3-4C8F-92C1-60B2D617A186}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OctodiffAsync", "OctodiffAsync\OctodiffAsync.csproj", "{77157EEA-0EDE-4DDA-A611-D57A8F013CD2}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastRsync.Benchmarks", "FastRsync.Benchmarks\FastRsync.Benchmarks.csproj", "{040122B6-214A-470A-BEB0-A2779E7EE3EC}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastRsync.Compression", "FastRsync.Compression\FastRsync.Compression.csproj", "{FA7C7A6D-68E6-445C-BD19-DD1FFBB34239}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastRsync.BackwardCompatibilityTests", "FastRsync.BackwardCompatibilityTests\FastRsync.BackwardCompatibilityTests.csproj", "{1F46EE18-00B8-49E4-8990-633410B807A2}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Debug|Any CPU.Build.0 = Debug|Any CPU {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Release|Any CPU.ActiveCfg = Release|Any CPU {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Release|Any CPU.Build.0 = Release|Any CPU {B7E95BD1-676E-4AC7-9089-3270392776BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B7E95BD1-676E-4AC7-9089-3270392776BA}.Debug|Any CPU.Build.0 = Debug|Any CPU {B7E95BD1-676E-4AC7-9089-3270392776BA}.Release|Any CPU.ActiveCfg = Release|Any CPU {B7E95BD1-676E-4AC7-9089-3270392776BA}.Release|Any CPU.Build.0 = Release|Any CPU {7B6474F5-8EE1-4D09-A48A-5A295E2DF8E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7B6474F5-8EE1-4D09-A48A-5A295E2DF8E2}.Debug|Any CPU.Build.0 = Debug|Any CPU {7B6474F5-8EE1-4D09-A48A-5A295E2DF8E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {7B6474F5-8EE1-4D09-A48A-5A295E2DF8E2}.Release|Any CPU.Build.0 = Release|Any CPU {CF4B2B1E-71C3-4C8F-92C1-60B2D617A186}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CF4B2B1E-71C3-4C8F-92C1-60B2D617A186}.Debug|Any CPU.Build.0 = Debug|Any CPU {CF4B2B1E-71C3-4C8F-92C1-60B2D617A186}.Release|Any CPU.ActiveCfg = Release|Any CPU {CF4B2B1E-71C3-4C8F-92C1-60B2D617A186}.Release|Any CPU.Build.0 = Release|Any CPU {77157EEA-0EDE-4DDA-A611-D57A8F013CD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {77157EEA-0EDE-4DDA-A611-D57A8F013CD2}.Debug|Any CPU.Build.0 = Debug|Any CPU {77157EEA-0EDE-4DDA-A611-D57A8F013CD2}.Release|Any CPU.ActiveCfg = Release|Any CPU {77157EEA-0EDE-4DDA-A611-D57A8F013CD2}.Release|Any CPU.Build.0 = Release|Any CPU {040122B6-214A-470A-BEB0-A2779E7EE3EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {040122B6-214A-470A-BEB0-A2779E7EE3EC}.Debug|Any CPU.Build.0 = Debug|Any CPU {040122B6-214A-470A-BEB0-A2779E7EE3EC}.Release|Any CPU.ActiveCfg = Release|Any CPU {040122B6-214A-470A-BEB0-A2779E7EE3EC}.Release|Any CPU.Build.0 = Release|Any CPU {FA7C7A6D-68E6-445C-BD19-DD1FFBB34239}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FA7C7A6D-68E6-445C-BD19-DD1FFBB34239}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA7C7A6D-68E6-445C-BD19-DD1FFBB34239}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA7C7A6D-68E6-445C-BD19-DD1FFBB34239}.Release|Any CPU.Build.0 = Release|Any CPU {1F46EE18-00B8-49E4-8990-633410B807A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1F46EE18-00B8-49E4-8990-633410B807A2}.Debug|Any CPU.Build.0 = Debug|Any CPU {1F46EE18-00B8-49E4-8990-633410B807A2}.Release|Any CPU.ActiveCfg = Release|Any CPU {1F46EE18-00B8-49E4-8990-633410B807A2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {224F70BE-E28E-4785-BF95-6E2F2B74893D} EndGlobalSection EndGlobal ================================================ FILE: source/Octodiff/CommandLine/DeltaCommand.cs ================================================ using System; using System.Collections.Generic; using System.IO; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using Octodiff.CommandLine.Support; namespace Octodiff.CommandLine { [Command("delta", Description = "Given a signature file and a new file, creates a delta file", Usage = " []")] class DeltaCommand : ICommand { private readonly List> configuration = new List>(); private readonly OptionSet options; private string newFilePath; private string signatureFilePath; private string deltaFilePath; public DeltaCommand() { options = new OptionSet(); options.Positional("signature-file", "The file containing the signature from the basis file.", v => signatureFilePath = v); options.Positional("new-file", "The file to create the delta from.", v => newFilePath = v); options.Positional("delta-file", "The file to write the delta to.", v => deltaFilePath = v); options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReport = new ConsoleProgressReporter())); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(signatureFilePath)) throw new OptionException("No signature file was specified", "new-file"); if (string.IsNullOrWhiteSpace(newFilePath)) throw new OptionException("No new file was specified", "new-file"); newFilePath = Path.GetFullPath(newFilePath); signatureFilePath = Path.GetFullPath(signatureFilePath); var delta = new DeltaBuilder(); foreach (var config in configuration) config(delta); if (!File.Exists(signatureFilePath)) { throw new FileNotFoundException("File not found: " + signatureFilePath, signatureFilePath); } if (!File.Exists(newFilePath)) { throw new FileNotFoundException("File not found: " + newFilePath, newFilePath); } if (string.IsNullOrWhiteSpace(deltaFilePath)) { deltaFilePath = newFilePath + ".octodelta"; } else { deltaFilePath = Path.GetFullPath(deltaFilePath); var directory = Path.GetDirectoryName(deltaFilePath); if (directory != null && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } } using (var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { delta.BuildDelta(newFileStream, new SignatureReader(signatureStream, delta.ProgressReport), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); } return 0; } } } ================================================ FILE: source/Octodiff/CommandLine/ExplainDeltaCommand.cs ================================================ using System; using System.IO; using System.Linq; using FastRsync.Delta; using Octodiff.CommandLine.Support; namespace Octodiff.CommandLine { [Command("explain-delta", Description = "Prints instructions from a delta file; useful when debugging.", Usage = "")] class ExplainDeltaCommand : ICommand { private readonly OptionSet options; private string deltaFilePath; public ExplainDeltaCommand() { options = new OptionSet(); options.Positional("delta-file", "The file to read the delta from.", v => deltaFilePath = v); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(deltaFilePath)) throw new OptionException("No delta file was specified", "delta-file"); deltaFilePath = Path.GetFullPath(deltaFilePath); if (!File.Exists(deltaFilePath)) { throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); } using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var reader = new BinaryDeltaReader(deltaStream, null); reader.Apply(data => { if (data.Length > 20) { Console.WriteLine("Data: ({0} bytes): {1}...", data.Length, BitConverter.ToString(data.Take(20).ToArray())); } else { Console.WriteLine("Data: ({0} bytes): {1}", data.Length, BitConverter.ToString(data.ToArray())); } }, (start, offset) => Console.WriteLine("Copy: {0:X} to {1:X}", start, offset)); } return 0; } } } ================================================ FILE: source/Octodiff/CommandLine/HelpCommand.cs ================================================ using System; using System.IO; using System.Linq; using Octodiff.CommandLine.Support; namespace Octodiff.CommandLine { [Command("help", "?", "h", Description = "Prints this help text")] class HelpCommand : ICommand { readonly ICommandLocator commands = new CommandLocator(); public void GetHelp(TextWriter writer) { } public int Execute(string[] commandLineArguments) { var executable = Path.GetFileNameWithoutExtension(new Uri(typeof(HelpCommand).Assembly.CodeBase).LocalPath); var commandName = commandLineArguments.FirstOrDefault(); if (string.IsNullOrEmpty(commandName)) { PrintGeneralHelp(executable); } else { var commandMeta = commands.Find(commandName); if (commandMeta == null) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Command '{0}' is not supported", commandName); Console.ResetColor(); PrintGeneralHelp(executable); } else { var command = commands.Create(commandMeta); PrintCommandHelp(executable, command, commandMeta, commandName); } } return 0; } void PrintCommandHelp(string executable, ICommand command, ICommandMetadata commandMetadata, string commandName) { Console.ResetColor(); Console.Write("Usage: "); Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(executable + " " + commandName + (!string.IsNullOrWhiteSpace(commandMetadata.Usage) ? " " + commandMetadata.Usage : "") + " []"); Console.ResetColor(); Console.WriteLine(); command.GetHelp(Console.Out); Console.WriteLine(); } void PrintGeneralHelp(string executable) { Console.ResetColor(); Console.Write("Usage: "); Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(executable + " "); Console.ResetColor(); Console.WriteLine(); Console.WriteLine("Where is one of: "); Console.WriteLine(); foreach (var possible in commands.List().OrderBy(x => x.Name)) { Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(" " + possible.Name.PadRight(15, ' ')); Console.ResetColor(); Console.WriteLine(" " + possible.Description); } Console.WriteLine(); Console.Write("Or use "); Console.ForegroundColor = ConsoleColor.White; Console.Write(executable + " help "); Console.ResetColor(); Console.WriteLine(" for more details."); } } } ================================================ FILE: source/Octodiff/CommandLine/PatchCommand.cs ================================================ using System; using System.IO; using FastRsync.Delta; using FastRsync.Diagnostics; using Octodiff.CommandLine.Support; namespace Octodiff.CommandLine { [Command("patch", Description = "Given a basis file, and a delta, produces the new file", Usage = " ")] class PatchCommand : ICommand { private readonly OptionSet options; private IProgress progressReport; private string basisFilePath; private string deltaFilePath; private string newFilePath; private bool skipHashCheck; public PatchCommand() { options = new OptionSet(); options.Positional("basis-file", "The file that the delta was created for.", v => basisFilePath = v); options.Positional("delta-file", "The delta to apply to the basis file", v => deltaFilePath = v); options.Positional("new-file", "The file to write the result to.", v => newFilePath = v); options.Add("progress", "Whether progress should be written to stdout", v => progressReport = new ConsoleProgressReporter()); options.Add("skip-verification", "Skip checking whether the basis file is the same as the file used to produce the signature that created the delta.", v => skipHashCheck = true); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(basisFilePath)) throw new OptionException("No basis file was specified", "basis-file"); if (string.IsNullOrWhiteSpace(deltaFilePath)) throw new OptionException("No delta file was specified", "delta-file"); if (string.IsNullOrWhiteSpace(newFilePath)) throw new OptionException("No new file was specified", "new-file"); basisFilePath = Path.GetFullPath(basisFilePath); deltaFilePath = Path.GetFullPath(deltaFilePath); newFilePath = Path.GetFullPath(newFilePath); if (!File.Exists(basisFilePath)) throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); if (!File.Exists(deltaFilePath)) throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); var directory = Path.GetDirectoryName(newFilePath); if (directory != null && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var delta = new DeltaApplier { SkipHashCheck = skipHashCheck }; using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var newFileStream = new FileStream(newFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) { delta.Apply(basisStream, new BinaryDeltaReader(deltaStream, progressReport), newFileStream); } return 0; } } } ================================================ FILE: source/Octodiff/CommandLine/SignatureCommand.cs ================================================ using System; using System.Collections.Generic; using System.IO; using FastRsync.Diagnostics; using FastRsync.Signature; using Octodiff.CommandLine.Support; namespace Octodiff.CommandLine { [Command("signature", "sig", Description = "Given a basis file, creates a signature file", Usage = " []")] class SignatureCommand : ICommand { private readonly List> configuration = new List>(); private readonly OptionSet options; private string basisFilePath; private string signatureFilePath; public SignatureCommand() { options = new OptionSet(); options.Positional("basis-file", "The file to read and create a signature from.", v => basisFilePath = v); options.Positional("signature-file", "The file to write the signature to.", v => signatureFilePath = v); options.Add("chunk-size=", string.Format("Maximum bytes per chunk. Defaults to {0}. Min of {1}, max of {2}.", SignatureBuilder.DefaultChunkSize, SignatureBuilder.MinimumChunkSize, SignatureBuilder.MaximumChunkSize), v => configuration.Add(builder => builder.ChunkSize = short.Parse(v))); options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReport = new ConsoleProgressReporter())); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(basisFilePath)) throw new OptionException("No basis file was specified", "basis-file"); basisFilePath = Path.GetFullPath(basisFilePath); var signatureBuilder = new SignatureBuilder(); foreach (var config in configuration) config(signatureBuilder); if (!File.Exists(basisFilePath)) { throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); } if (string.IsNullOrWhiteSpace(signatureFilePath)) { signatureFilePath = basisFilePath + ".octosig"; } else { signatureFilePath = Path.GetFullPath(signatureFilePath); var sigDirectory = Path.GetDirectoryName(signatureFilePath); if (sigDirectory != null && !Directory.Exists(sigDirectory)) { Directory.CreateDirectory(sigDirectory); } } using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { signatureBuilder.Build(basisStream, new SignatureWriter(signatureStream)); } return 0; } } } ================================================ FILE: source/Octodiff/CommandLine/Support/CommandAttribute.cs ================================================ using System; namespace Octodiff.CommandLine.Support { [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] class CommandAttribute : Attribute, ICommandMetadata { public CommandAttribute(string name, params string[] aliases) { Name = name; Aliases = aliases; } public string Name { get; set; } public string[] Aliases { get; set; } public string Description { get; set; } public string Usage { get; set; } } } ================================================ FILE: source/Octodiff/CommandLine/Support/CommandException.cs ================================================ using System; namespace Octodiff.CommandLine.Support { class CommandException : Exception { public CommandException(string message) : base(message) { } } } ================================================ FILE: source/Octodiff/CommandLine/Support/CommandLocator.cs ================================================ using System; using System.Linq; namespace Octodiff.CommandLine.Support { class CommandLocator : ICommandLocator { public ICommandMetadata[] List() { return (from t in typeof (CommandLocator).Assembly.GetTypes() where typeof (ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata) t.GetCustomAttributes(typeof (CommandAttribute), true).FirstOrDefault() where attribute != null select attribute).ToArray(); } public ICommandMetadata Find(string name) { name = name.Trim().ToLowerInvariant(); return (from t in typeof (CommandLocator).Assembly.GetTypes() where typeof (ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata) t.GetCustomAttributes(typeof (CommandAttribute), true).FirstOrDefault() where attribute != null where attribute.Name == name || attribute.Aliases.Any(a => a == name) select attribute).FirstOrDefault(); } public ICommand Create(ICommandMetadata metadata) { var name = metadata.Name; var found = (from t in typeof(CommandLocator).Assembly.GetTypes() where typeof(ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata)t.GetCustomAttributes(typeof(CommandAttribute), true).FirstOrDefault() where attribute != null where attribute.Name == name || attribute.Aliases.Any(a => a == name) select t).FirstOrDefault(); return found == null ? null : (ICommand)Activator.CreateInstance(found); } } } ================================================ FILE: source/Octodiff/CommandLine/Support/ICommand.cs ================================================ using System.IO; namespace Octodiff.CommandLine.Support { interface ICommand { void GetHelp(TextWriter writer); int Execute(string[] commandLineArguments); } } ================================================ FILE: source/Octodiff/CommandLine/Support/ICommandLocator.cs ================================================ namespace Octodiff.CommandLine.Support { interface ICommandLocator { ICommandMetadata[] List(); ICommandMetadata Find(string name); ICommand Create(ICommandMetadata metadata); } } ================================================ FILE: source/Octodiff/CommandLine/Support/ICommandMetadata.cs ================================================ namespace Octodiff.CommandLine.Support { interface ICommandMetadata { string Name { get; } string[] Aliases { get; } string Description { get; } string Usage { get; set; } } } ================================================ FILE: source/Octodiff/CommandLine/Support/NDesk.Options.cs ================================================ // // Options.cs // // Authors: // Jonathan Pryor // // Copyright (C) 2008 Novell (http://www.novell.com) // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // Compile With: // gmcs -debug+ -r:System.Core Options.cs -o:NDesk.Options.dll // gmcs -debug+ -d:LINQ -r:System.Core Options.cs -o:NDesk.Options.dll // // The LINQ version just changes the implementation of // OptionSet.Parse(IEnumerable), and confers no semantic changes. // // A Getopt::Long-inspired option parsing library for C#. // // NDesk.Options.OptionSet is built upon a key/value table, where the // key is a option format string and the value is a delegate that is // invoked when the format string is matched. // // Option format strings: // Regex-like BNF Grammar: // name: .+ // type: [=:] // sep: ( [^{}]+ | '{' .+ '}' )? // aliases: ( name type sep ) ( '|' name type sep )* // // Each '|'-delimited name is an alias for the associated action. If the // format string ends in a '=', it has a required value. If the format // string ends in a ':', it has an optional value. If neither '=' or ':' // is present, no value is supported. `=' or `:' need only be defined on one // alias, but if they are provided on more than one they must be consistent. // // Each alias portion may also end with a "key/value separator", which is used // to split option values if the option accepts > 1 value. If not specified, // it defaults to '=' and ':'. If specified, it can be any character except // '{' and '}' OR the *string* between '{' and '}'. If no separator should be // used (i.e. the separate values should be distinct arguments), then "{}" // should be used as the separator. // // Options are extracted either from the current option by looking for // the option name followed by an '=' or ':', or is taken from the // following option IFF: // - The current option does not contain a '=' or a ':' // - The current option requires a value (i.e. not a Option type of ':') // // The `name' used in the option format string does NOT include any leading // option indicator, such as '-', '--', or '/'. All three of these are // permitted/required on any named option. // // Option bundling is permitted so long as: // - '-' is used to start the option group // - all of the bundled options are a single character // - at most one of the bundled options accepts a value, and the value // provided starts from the next character to the end of the string. // // This allows specifying '-a -b -c' as '-abc', and specifying '-D name=value' // as '-Dname=value'. // // Option processing is disabled by specifying "--". All options after "--" // are returned by OptionSet.Parse() unchanged and unprocessed. // // Unprocessed options are returned from OptionSet.Parse(). // // Examples: // int verbose = 0; // OptionSet p = new OptionSet () // .Add ("v", v => ++verbose) // .Add ("name=|value=", v => Console.WriteLine (v)); // p.Parse (new string[]{"-v", "--v", "/v", "-name=A", "/name", "B", "extra"}); // // The above would parse the argument string array, and would invoke the // lambda expression three times, setting `verbose' to 3 when complete. // It would also print out "A" and "B" to standard output. // The returned array would contain the string "extra". // // C# 3.0 collection initializers are supported and encouraged: // var p = new OptionSet () { // { "h|?|help", v => ShowHelp () }, // }; // // System.ComponentModel.TypeConverter is also supported, allowing the use of // custom data types in the callback type; TypeConverter.ConvertFromString() // is used to convert the value option to an instance of the specified // type: // // var p = new OptionSet () { // { "foo=", (Foo f) => Console.WriteLine (f.ToString ()) }, // }; // // Random other tidbits: // - Boolean options (those w/o '=' or ':' in the option format string) // are explicitly enabled if they are followed with '+', and explicitly // disabled if they are followed with '-': // string a = null; // var p = new OptionSet () { // { "a", s => a = s }, // }; // p.Parse (new string[]{"-a"}); // sets v != null // p.Parse (new string[]{"-a+"}); // sets v != null // p.Parse (new string[]{"-a-"}); // sets v == null // using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.ComponentModel; using System.IO; using System.Linq; using System.Runtime.Serialization; using System.Security.Permissions; using System.Text; using System.Text.RegularExpressions; #if LINQ using System.Linq; #endif #if TEST using NDesk.Options; #endif #if NDESK_OPTIONS namespace NDesk.Options #else namespace Octodiff.CommandLine #endif { class OptionValueCollection : IList, IList { List values = new List(); OptionContext c; internal OptionValueCollection(OptionContext c) { this.c = c; } #region ICollection void ICollection.CopyTo(Array array, int index) { (values as ICollection).CopyTo(array, index); } bool ICollection.IsSynchronized { get { return (values as ICollection).IsSynchronized; } } object ICollection.SyncRoot { get { return (values as ICollection).SyncRoot; } } #endregion #region ICollection public void Add(string item) { values.Add(item); } public void Clear() { values.Clear(); } public bool Contains(string item) { return values.Contains(item); } public void CopyTo(string[] array, int arrayIndex) { values.CopyTo(array, arrayIndex); } public bool Remove(string item) { return values.Remove(item); } public int Count { get { return values.Count; } } public bool IsReadOnly { get { return false; } } #endregion #region IEnumerable IEnumerator IEnumerable.GetEnumerator() { return values.GetEnumerator(); } #endregion #region IEnumerable public IEnumerator GetEnumerator() { return values.GetEnumerator(); } #endregion #region IList int IList.Add(object value) { return (values as IList).Add(value); } bool IList.Contains(object value) { return (values as IList).Contains(value); } int IList.IndexOf(object value) { return (values as IList).IndexOf(value); } void IList.Insert(int index, object value) { (values as IList).Insert(index, value); } void IList.Remove(object value) { (values as IList).Remove(value); } void IList.RemoveAt(int index) { (values as IList).RemoveAt(index); } bool IList.IsFixedSize { get { return false; } } object IList.this[int index] { get { return this[index]; } set { (values as IList)[index] = value; } } #endregion #region IList public int IndexOf(string item) { return values.IndexOf(item); } public void Insert(int index, string item) { values.Insert(index, item); } public void RemoveAt(int index) { values.RemoveAt(index); } private void AssertValid(int index) { if (c.Option == null) throw new InvalidOperationException("OptionContext.Option is null."); if (index >= c.Option.MaxValueCount) throw new ArgumentOutOfRangeException("index"); if (c.Option.OptionValueType == OptionValueType.Required && index >= values.Count) throw new OptionException(string.Format( c.OptionSet.MessageLocalizer("Missing required value for option '{0}'."), c.OptionName), c.OptionName); } public string this[int index] { get { AssertValid(index); return index >= values.Count ? null : values[index]; } set { values[index] = value; } } #endregion public List ToList() { return new List(values); } public string[] ToArray() { return values.ToArray(); } public override string ToString() { return string.Join(", ", values.ToArray()); } } class OptionContext { private Option option; private string name; private int index; private OptionSet set; private OptionValueCollection c; public OptionContext(OptionSet set) { this.set = set; this.c = new OptionValueCollection(this); } public Option Option { get { return option; } set { option = value; } } public string OptionName { get { return name; } set { name = value; } } public int OptionIndex { get { return index; } set { index = value; } } public OptionSet OptionSet { get { return set; } } public OptionValueCollection OptionValues { get { return c; } } } enum OptionValueType { None, Optional, Required, } abstract class Option { string prototype, description; string[] names; OptionValueType type; int count; string[] separators; protected Option(string prototype, string description) : this(prototype, description, 1) { } protected Option(string prototype, string description, int maxValueCount) { if (prototype == null) throw new ArgumentNullException("prototype"); if (prototype.Length == 0) throw new ArgumentException("Cannot be the empty string.", "prototype"); if (maxValueCount < 0) throw new ArgumentOutOfRangeException("maxValueCount"); this.prototype = prototype; this.names = prototype.Split('|'); this.description = description; this.count = maxValueCount; this.type = ParsePrototype(); if (this.count == 0 && type != OptionValueType.None) throw new ArgumentException( "Cannot provide maxValueCount of 0 for OptionValueType.Required or " + "OptionValueType.Optional.", "maxValueCount"); if (this.type == OptionValueType.None && maxValueCount > 1) throw new ArgumentException( string.Format("Cannot provide maxValueCount of {0} for OptionValueType.None.", maxValueCount), "maxValueCount"); if (Array.IndexOf(names, "<>") >= 0 && ((names.Length == 1 && this.type != OptionValueType.None) || (names.Length > 1 && this.MaxValueCount > 1))) throw new ArgumentException( "The default option handler '<>' cannot require values.", "prototype"); } public string Prototype { get { return prototype; } } public string Description { get { return description; } } public OptionValueType OptionValueType { get { return type; } } public int MaxValueCount { get { return count; } } public string[] GetNames() { return (string[])names.Clone(); } public string[] GetValueSeparators() { if (separators == null) return new string[0]; return (string[])separators.Clone(); } protected static T Parse(string value, OptionContext c) { Type tt = typeof(T); bool nullable = tt.IsValueType && tt.IsGenericType && !tt.IsGenericTypeDefinition && tt.GetGenericTypeDefinition() == typeof(Nullable<>); Type targetType = nullable ? tt.GetGenericArguments()[0] : typeof(T); TypeConverter conv = TypeDescriptor.GetConverter(targetType); T t = default(T); try { if (value != null) t = (T)conv.ConvertFromString(value); } catch (Exception e) { throw new OptionException( string.Format( c.OptionSet.MessageLocalizer("Could not convert string `{0}' to type {1} for option `{2}'."), value, targetType.Name, c.OptionName), c.OptionName, e); } return t; } internal string[] Names { get { return names; } } internal string[] ValueSeparators { get { return separators; } } static readonly char[] NameTerminator = new char[] { '=', ':' }; private OptionValueType ParsePrototype() { char type = '\0'; List seps = new List(); for (int i = 0; i < names.Length; ++i) { string name = names[i]; if (name.Length == 0) throw new ArgumentException("Empty option names are not supported.", "prototype"); int end = name.IndexOfAny(NameTerminator); if (end == -1) continue; names[i] = name.Substring(0, end); if (type == '\0' || type == name[end]) type = name[end]; else throw new ArgumentException( string.Format("Conflicting option types: '{0}' vs. '{1}'.", type, name[end]), "prototype"); AddSeparators(name, end, seps); } if (type == '\0') return OptionValueType.None; if (count <= 1 && seps.Count != 0) throw new ArgumentException( string.Format("Cannot provide key/value separators for Options taking {0} value(s).", count), "prototype"); if (count > 1) { if (seps.Count == 0) this.separators = new string[] { ":", "=" }; else if (seps.Count == 1 && seps[0].Length == 0) this.separators = null; else this.separators = seps.ToArray(); } return type == '=' ? OptionValueType.Required : OptionValueType.Optional; } private static void AddSeparators(string name, int end, ICollection seps) { int start = -1; for (int i = end + 1; i < name.Length; ++i) { switch (name[i]) { case '{': if (start != -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); start = i + 1; break; case '}': if (start == -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); seps.Add(name.Substring(start, i - start)); start = -1; break; default: if (start == -1) seps.Add(name[i].ToString()); break; } } if (start != -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); } public void Invoke(OptionContext c) { OnParseComplete(c); c.OptionName = null; c.Option = null; c.OptionValues.Clear(); } protected abstract void OnParseComplete(OptionContext c); public override string ToString() { return Prototype; } } [Serializable] class OptionException : Exception { private string option; public OptionException() { } public OptionException(string message, string optionName) : base(message) { this.option = optionName; } public OptionException(string message, string optionName, Exception innerException) : base(message, innerException) { this.option = optionName; } protected OptionException(SerializationInfo info, StreamingContext context) : base(info, context) { this.option = info.GetString("OptionName"); } public string OptionName { get { return this.option; } } [SecurityPermission(SecurityAction.LinkDemand, SerializationFormatter = true)] public override void GetObjectData(SerializationInfo info, StreamingContext context) { base.GetObjectData(info, context); info.AddValue("OptionName", option); } } delegate void OptionAction(TKey key, TValue value); class OptionSet : KeyedCollection { readonly List>> positionalParameters = new List>>(); public OptionSet() : this(delegate(string f) { return f; }) { } public OptionSet(Converter localizer) { this.localizer = localizer; } Converter localizer; public Converter MessageLocalizer { get { return localizer; } } protected override string GetKeyForItem(Option item) { if (item == null) throw new ArgumentNullException("option"); if (item.Names != null && item.Names.Length > 0) return item.Names[0]; // This should never happen, as it's invalid for Option to be // constructed w/o any names. throw new InvalidOperationException("Option has no names!"); } [Obsolete("Use KeyedCollection.this[string]")] protected Option GetOptionForName(string option) { if (option == null) throw new ArgumentNullException("option"); try { return base[option]; } catch (KeyNotFoundException) { return null; } } protected override void InsertItem(int index, Option item) { base.InsertItem(index, item); AddImpl(item); } protected override void RemoveItem(int index) { base.RemoveItem(index); Option p = Items[index]; // KeyedCollection.RemoveItem() handles the 0th item for (int i = 1; i < p.Names.Length; ++i) { Dictionary.Remove(p.Names[i]); } } protected override void SetItem(int index, Option item) { base.SetItem(index, item); RemoveItem(index); AddImpl(item); } private void AddImpl(Option option) { if (option == null) throw new ArgumentNullException("option"); List added = new List(option.Names.Length); try { // KeyedCollection.InsertItem/SetItem handle the 0th name. for (int i = 1; i < option.Names.Length; ++i) { Dictionary.Add(option.Names[i], option); added.Add(option.Names[i]); } } catch (Exception) { foreach (string name in added) Dictionary.Remove(name); throw; } } public new OptionSet Add(Option option) { base.Add(option); return this; } public void Positional(string name, string description, Action valueCallback) { positionalParameters.Add(Tuple.Create(name, description, valueCallback)); } sealed class ActionOption : Option { Action action; public ActionOption(string prototype, string description, int count, Action action) : base(prototype, description, count) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action(c.OptionValues); } } public OptionSet Add(string prototype, Action action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, Action action) { if (action == null) throw new ArgumentNullException("action"); Option p = new ActionOption(prototype, description, 1, delegate(OptionValueCollection v) { action(v[0]); }); base.Add(p); return this; } public OptionSet Add(string prototype, OptionAction action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, OptionAction action) { if (action == null) throw new ArgumentNullException("action"); Option p = new ActionOption(prototype, description, 2, delegate(OptionValueCollection v) { action(v[0], v[1]); }); base.Add(p); return this; } sealed class ActionOption : Option { Action action; public ActionOption(string prototype, string description, Action action) : base(prototype, description, 1) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action(Parse(c.OptionValues[0], c)); } } sealed class ActionOption : Option { OptionAction action; public ActionOption(string prototype, string description, OptionAction action) : base(prototype, description, 2) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action( Parse(c.OptionValues[0], c), Parse(c.OptionValues[1], c)); } } public OptionSet Add(string prototype, Action action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, Action action) { return Add(new ActionOption(prototype, description, action)); } public OptionSet Add(string prototype, OptionAction action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, OptionAction action) { return Add(new ActionOption(prototype, description, action)); } protected virtual OptionContext CreateOptionContext() { return new OptionContext(this); } #if LINQ public List Parse (IEnumerable arguments) { bool process = true; OptionContext c = CreateOptionContext (); c.OptionIndex = -1; var def = GetOptionForName ("<>"); var unprocessed = from argument in arguments where ++c.OptionIndex >= 0 && (process || def != null) ? process ? argument == "--" ? (process = false) : !Parse (argument, c) ? def != null ? Unprocessed (null, def, c, argument) : true : false : def != null ? Unprocessed (null, def, c, argument) : true : true select argument; List r = unprocessed.ToList (); if (c.Option != null) c.Option.Invoke (c); return r; } #else public List Parse(IEnumerable arguments) { OptionContext c = CreateOptionContext(); c.OptionIndex = -1; bool process = true; List unprocessed = new List(); Option def = Contains("<>") ? this["<>"] : null; foreach (string argument in arguments) { ++c.OptionIndex; if (argument == "--") { process = false; continue; } if (!process) { Unprocessed(unprocessed, def, c, argument); continue; } if (!Parse(argument, c)) Unprocessed(unprocessed, def, c, argument); } if (c.Option != null) c.Option.Invoke(c); for (var i = 0; i < positionalParameters.Count; i++) { if (i < unprocessed.Count) { positionalParameters[i].Item3(unprocessed[i]); } } return unprocessed; } #endif private static bool Unprocessed(ICollection extra, Option def, OptionContext c, string argument) { if (def == null) { extra.Add(argument); return false; } c.OptionValues.Add(argument); c.Option = def; c.Option.Invoke(c); return false; } private readonly Regex ValueOption = new Regex( @"^(?--|-|/)(?[^:=]+)((?[:=])(?.*))?$"); protected bool GetOptionParts(string argument, out string flag, out string name, out string sep, out string value) { if (argument == null) throw new ArgumentNullException("argument"); flag = name = sep = value = null; Match m = ValueOption.Match(argument); if (!m.Success) { return false; } flag = m.Groups["flag"].Value; name = m.Groups["name"].Value; if (m.Groups["sep"].Success && m.Groups["value"].Success) { sep = m.Groups["sep"].Value; value = m.Groups["value"].Value; } return true; } protected virtual bool Parse(string argument, OptionContext c) { if (c.Option != null) { ParseValue(argument, c); return true; } string f, n, s, v; if (!GetOptionParts(argument, out f, out n, out s, out v)) return false; Option p; if (Contains(n)) { p = this[n]; c.OptionName = f + n; c.Option = p; switch (p.OptionValueType) { case OptionValueType.None: c.OptionValues.Add(n); c.Option.Invoke(c); break; case OptionValueType.Optional: case OptionValueType.Required: ParseValue(v, c); break; } return true; } // no match; is it a bool option? if (ParseBool(argument, n, c)) return true; // is it a bundled option? if (ParseBundledValue(f, string.Concat(n + s + v), c)) return true; return false; } private void ParseValue(string option, OptionContext c) { if (option != null) foreach (string o in c.Option.ValueSeparators != null ? option.Split(c.Option.ValueSeparators, StringSplitOptions.None) : new string[] { option }) { c.OptionValues.Add(o); } if (c.OptionValues.Count == c.Option.MaxValueCount || c.Option.OptionValueType == OptionValueType.Optional) c.Option.Invoke(c); else if (c.OptionValues.Count > c.Option.MaxValueCount) { throw new OptionException(localizer(string.Format( "Error: Found {0} option values when expecting {1}.", c.OptionValues.Count, c.Option.MaxValueCount)), c.OptionName); } } private bool ParseBool(string option, string n, OptionContext c) { Option p; string rn; if (n.Length >= 1 && (n[n.Length - 1] == '+' || n[n.Length - 1] == '-') && Contains((rn = n.Substring(0, n.Length - 1)))) { p = this[rn]; string v = n[n.Length - 1] == '+' ? option : null; c.OptionName = option; c.Option = p; c.OptionValues.Add(v); p.Invoke(c); return true; } return false; } private bool ParseBundledValue(string f, string n, OptionContext c) { if (f != "-") return false; for (int i = 0; i < n.Length; ++i) { Option p; string opt = f + n[i].ToString(); string rn = n[i].ToString(); if (!Contains(rn)) { if (i == 0) return false; throw new OptionException(string.Format(localizer( "Cannot bundle unregistered option '{0}'."), opt), opt); } p = this[rn]; switch (p.OptionValueType) { case OptionValueType.None: Invoke(c, opt, n, p); break; case OptionValueType.Optional: case OptionValueType.Required: { string v = n.Substring(i + 1); c.Option = p; c.OptionName = opt; ParseValue(v.Length != 0 ? v : null, c); return true; } default: throw new InvalidOperationException("Unknown OptionValueType: " + p.OptionValueType); } } return true; } private static void Invoke(OptionContext c, string name, string value, Option option) { c.OptionName = name; c.Option = option; c.OptionValues.Add(value); option.Invoke(c); } private const int OptionWidth = 29; public List GetPositionals() { return positionalParameters.Select(s => s.Item1).ToList(); } public void WriteOptionDescriptions(TextWriter o) { if (positionalParameters.Count > 0) { o.WriteLine("Arguments:"); o.WriteLine(); } foreach (var positional in positionalParameters) { var prototype = " " + positional.Item1; o.Write(prototype); var written = prototype.Length; if (written < OptionWidth) o.Write(new string(' ', OptionWidth - written)); else { o.WriteLine(); o.Write(new string(' ', OptionWidth)); } bool indent = false; string prefix = new string(' ', OptionWidth); foreach (string line in GetLines(positional.Item2)) { if (indent) o.Write(prefix); o.WriteLine(line); indent = true; } } if (this.Count > 0) { if (positionalParameters.Count > 0) { o.WriteLine(); } o.WriteLine("Options:"); o.WriteLine(); } foreach (Option p in this) { int written = 0; if (!WriteOptionPrototype(o, p, ref written)) continue; if (written < OptionWidth) o.Write(new string(' ', OptionWidth - written)); else { o.WriteLine(); o.Write(new string(' ', OptionWidth)); } bool indent = false; string prefix = new string(' ', OptionWidth); foreach (string line in GetLines(localizer(GetDescription(p.Description)))) { if (indent) o.Write(prefix); o.WriteLine(line); indent = true; } } } bool WriteOptionPrototype(TextWriter o, Option p, ref int written) { string[] names = p.Names; int i = GetNextOptionIndex(names, 0); if (i == names.Length) return false; if (names[i].Length == 1) { Write(o, ref written, " -"); Write(o, ref written, names[0]); } else { Write(o, ref written, " --"); Write(o, ref written, names[0]); } for (i = GetNextOptionIndex(names, i + 1); i < names.Length; i = GetNextOptionIndex(names, i + 1)) { Write(o, ref written, ", "); Write(o, ref written, names[i].Length == 1 ? "-" : "--"); Write(o, ref written, names[i]); } if (p.OptionValueType == OptionValueType.Optional || p.OptionValueType == OptionValueType.Required) { if (p.OptionValueType == OptionValueType.Optional) { Write(o, ref written, localizer("[")); } Write(o, ref written, localizer("=" + GetArgumentName(0, p.MaxValueCount, p.Description))); string sep = p.ValueSeparators != null && p.ValueSeparators.Length > 0 ? p.ValueSeparators[0] : " "; for (int c = 1; c < p.MaxValueCount; ++c) { Write(o, ref written, localizer(sep + GetArgumentName(c, p.MaxValueCount, p.Description))); } if (p.OptionValueType == OptionValueType.Optional) { Write(o, ref written, localizer("]")); } } return true; } static int GetNextOptionIndex(string[] names, int i) { while (i < names.Length && names[i] == "<>") { ++i; } return i; } static void Write(TextWriter o, ref int n, string s) { n += s.Length; o.Write(s); } private static string GetArgumentName(int index, int maxIndex, string description) { if (description == null) return maxIndex == 1 ? "VALUE" : "VALUE" + (index + 1); string[] nameStart; if (maxIndex == 1) nameStart = new string[] { "{0:", "{" }; else nameStart = new string[] { "{" + index + ":" }; for (int i = 0; i < nameStart.Length; ++i) { int start, j = 0; do { start = description.IndexOf(nameStart[i], j); } while (start >= 0 && j != 0 ? description[j++ - 1] == '{' : false); if (start == -1) continue; int end = description.IndexOf("}", start); if (end == -1) continue; return description.Substring(start + nameStart[i].Length, end - start - nameStart[i].Length); } return maxIndex == 1 ? "VALUE" : "VALUE" + (index + 1); } private static string GetDescription(string description) { if (description == null) return string.Empty; StringBuilder sb = new StringBuilder(description.Length); int start = -1; for (int i = 0; i < description.Length; ++i) { switch (description[i]) { case '{': if (i == start) { sb.Append('{'); start = -1; } else if (start < 0) start = i + 1; break; case '}': if (start < 0) { if ((i + 1) == description.Length || description[i + 1] != '}') throw new InvalidOperationException("Invalid option description: " + description); ++i; sb.Append("}"); } else { sb.Append(description.Substring(start, i - start)); start = -1; } break; case ':': if (start < 0) goto default; start = i + 1; break; default: if (start < 0) sb.Append(description[i]); break; } } return sb.ToString(); } private static IEnumerable GetLines(string description) { if (string.IsNullOrEmpty(description)) { yield return string.Empty; yield break; } int length = 80 - OptionWidth - 1; int start = 0, end; do { end = GetLineEnd(start, length, description); char c = description[end - 1]; if (char.IsWhiteSpace(c)) --end; bool writeContinuation = end != description.Length && !IsEolChar(c); string line = description.Substring(start, end - start) + (writeContinuation ? "-" : ""); yield return line; start = end; if (char.IsWhiteSpace(c)) ++start; length = 80 - OptionWidth - 2 - 1; } while (end < description.Length); } private static bool IsEolChar(char c) { return !char.IsLetterOrDigit(c); } private static int GetLineEnd(int start, int length, string description) { int end = System.Math.Min(start + length, description.Length); int sep = -1; for (int i = start + 1; i < end; ++i) { if (description[i] == '\n') return i + 1; if (IsEolChar(description[i])) sep = i + 1; } if (sep == -1 || end == description.Length) return end; return sep; } } } ================================================ FILE: source/Octodiff/Octodiff.csproj ================================================  net472 7.3 Exe false false false false false false false false false AnyCPU true full false bin\ DEBUG;TRACE prompt 4 bin\Octodiff.XML false AnyCPU pdbonly true bin\ TRACE prompt 4 bin\Octodiff.XML false ================================================ FILE: source/Octodiff/OctodiffProgram.cs ================================================ using System; using System.Collections.Generic; using System.IO; using System.Linq; using Octodiff.CommandLine; using Octodiff.CommandLine.Support; namespace Octodiff { public class OctodiffProgram { static int Main(string[] args) { string[] commandArguments; var commandName = ExtractCommand(args, out commandArguments); var locator = new CommandLocator(); var command = locator.Find(commandName); if (command == null) { locator.Create(locator.Find("help")).Execute(commandArguments); return 4; } try { var exitCode = locator.Create(command).Execute(commandArguments); return exitCode; } catch (OptionException ex) { WriteError(ex); locator.Create(locator.Find("help")).Execute(new[] {commandName}); return 4; } catch (ArgumentException ex) { WriteError(ex); return 4; } catch (FileNotFoundException ex) { WriteError(ex); return 4; } catch (InvalidDataException ex) { WriteError(ex); return 2; } catch (IOException ex) { WriteError(ex, details: true); return 1; } catch (Exception ex) { WriteError(ex, details: true); return 3; } } static void WriteError(Exception ex, bool details = false) { Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Error: " + ex.Message); Console.ResetColor(); if (details) { Console.WriteLine(ex.ToString()); } } private static string ExtractCommand(ICollection args, out string[] remaining) { remaining = args.Count <= 1 ? new string[0] : args.Skip(1).ToArray(); return (args.FirstOrDefault() ?? string.Empty).ToLowerInvariant(); } } } ================================================ FILE: source/Octodiff/Properties/AssemblyInfo.cs ================================================ using System.Reflection; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("Octodiff")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] [assembly: AssemblyProduct("Octodiff")] [assembly: AssemblyCopyright("Copyright © 2014")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("4c0d9c4b-8a4d-4488-9a83-553a046c18b6")] // Version information for an assembly consists of the following four values: // // Major Version // Minor Version // Build Number // Revision // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] [assembly: AssemblyVersion("1.0.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] ================================================ FILE: source/Octodiff/app.config ================================================ ================================================ FILE: source/Octodiff.Tests/DeltaFixture.cs ================================================ using System; using System.Diagnostics; using System.IO; using System.Text.RegularExpressions; using NUnit.Framework; using Octodiff.Tests.Util; using NUnit.Framework.Legacy; namespace Octodiff.Tests { [TestFixture] public class DeltaFixture : CommandLineFixture { [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void DeltaOfUnchangedFileShouldResultInJustCopySegment(string name, int numberOfFiles, OctodiffAppVariant octodiff) { PackageGenerator.GeneratePackage(name, numberOfFiles); Run("signature " + name + " " + name + ".sig", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); Run("delta " + name + ".sig " + name + " " + name + ".delta", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); Run("explain-delta " + name + ".delta", octodiff); Assert.That(Regex.IsMatch(Output, "^Copy: 0 to ([0-9A-F]+)\r\n$")); Assert.That(Output, Does.Not.Contains("Data:")); } [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void DeltaOfChangedFileShouldResultInNewDataSegments(string name, int numberOfFiles, OctodiffAppVariant octodiff) { PackageGenerator.GeneratePackage(name, numberOfFiles); Run("signature " + name + " " + name + ".sig", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); var newName = Path.ChangeExtension(name, "2.zip"); PackageGenerator.ModifyPackage(name, newName, (int) (0.33*numberOfFiles), (int) (0.10*numberOfFiles)); Run("delta " + name + ".sig " + newName + " " + name + ".delta", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); Run("explain-delta " + name + ".delta", octodiff); Assert.That(Regex.IsMatch(Output, "Copy: ([0-9A-F]+) to ([0-9A-F]+)\r\n")); Assert.That(Regex.IsMatch(Output, "Data: \\(([0-9]+) bytes\\)")); var originalSize = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, name)).Length; var newSize = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, newName)).Length; var deltaSize = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, name + ".delta")).Length; var actualDifference = Math.Abs(newSize - originalSize); var deltaToActualRatio = (double) deltaSize/actualDifference; Trace.WriteLine(string.Format("Delta ratio: {0:n3}", deltaToActualRatio)); Assert.That(deltaSize * 2 < newSize, Is.True, "Delta should be at least half the new file size"); Assert.That(0.80 <= deltaToActualRatio && deltaToActualRatio <= 1.60, Is.True, "Delta should be pretty close to the actual file differences"); } } } ================================================ FILE: source/Octodiff.Tests/HelpFixture.cs ================================================ using NUnit.Framework; using Octodiff.Tests.Util; namespace Octodiff.Tests { [TestFixture] public class HelpFixture : CommandLineFixture { [Test] [TestCase(4, "")] [TestCase(0, "help")] [TestCase(4, "foo")] public void ShouldPrintGeneralHelp(int exitCode, string args) { Run(args, OctodiffAppVariant.Sync); Assert.That(ExitCode, Is.EqualTo(exitCode)); Assert.That(Output, Does.Contain("Usage: Octodiff ")); Assert.That(Output, Does.Not.Contain("Error")); } [Test] [TestCase(0, "help signature", "signature")] [TestCase(0, "help delta", "delta")] [TestCase(0, "help patch", "patch")] [TestCase(0, "help explain-delta", "explain-delta")] public void ShouldPrintCommandHelp(int exitCode, string args, string commandName) { Run(args, OctodiffAppVariant.Sync); Assert.That(ExitCode, Is.EqualTo(exitCode)); Assert.That(Output, Does.Contain("Usage: Octodiff " + commandName)); Assert.That(Output, Does.Contain("Usage: Octodiff " + commandName)); Assert.That(Output, Does.Not.Contain("Error")); } [Test] [TestCase(4, "signature", "No basis file was specified")] [TestCase(4, "delta", "No signature file was specified")] [TestCase(4, "delta foo.sig", "No new file was specified")] [TestCase(4, "patch", "No basis file was specified")] [TestCase(4, "patch foo.nupkg", "No delta file was specified")] [TestCase(4, "patch foo.nupkg foo.delta", "No new file was specified")] public void ShouldPrintHelpWhenAllArgumentsAreNotSpecified(int exitCode, string args, string text) { Run(args, OctodiffAppVariant.Sync); Assert.That(ExitCode, Is.EqualTo(exitCode)); Assert.That(Output, Does.Contain("Usage")); Assert.That(Output, Does.Contain("Error: " + text)); } } } ================================================ FILE: source/Octodiff.Tests/Octodiff.Tests.csproj ================================================  net472 7.3 false false false false false false false false false true full false bin\Debug\ DEBUG;TRACE prompt 4 false pdbonly true bin\Release\ TRACE prompt 4 false ================================================ FILE: source/Octodiff.Tests/PackageGenerator.cs ================================================ using System; using System.IO; using System.IO.Packaging; using System.Linq; using NUnit.Framework; namespace Octodiff.Tests { public class PackageGenerator { static Random r = new Random(); public static void GeneratePackage(string fileName, int numberOfFiles = 10, int averageFileSize = 100*1024) { var fullPath = Path.Combine(TestContext.CurrentContext.TestDirectory, fileName); using (var package = ZipPackage.Open(fullPath, FileMode.Create)) { for (int i = 0; i < numberOfFiles; i++) { var buffer = new byte[averageFileSize]; var part = package.CreatePart(new Uri("/" + Guid.NewGuid(), UriKind.Relative), "text/plain"); using (var partStream = part.GetStream(FileMode.Create)) { r.NextBytes(buffer); partStream.Write(buffer, 0, buffer.Length); } } } } public static void ModifyPackage(string fileName, string newFileName, int filesToAdd, int filesToRemove, int averageFileSize = 100*1024) { var oldFilePath = Path.Combine(TestContext.CurrentContext.TestDirectory, fileName); var fullPath = Path.Combine(TestContext.CurrentContext.TestDirectory, newFileName); File.Copy(oldFilePath, fullPath, true); using (var package = ZipPackage.Open(fullPath, FileMode.Open)) { for (int i = 0; i < filesToAdd; i++) { var buffer = new byte[averageFileSize]; var part = package.CreatePart(new Uri("/" + Guid.NewGuid(), UriKind.Relative), "text/plain"); using (var partStream = part.GetStream(FileMode.Create)) { r.NextBytes(buffer); partStream.Write(buffer, 0, buffer.Length); } } var parts = package.GetParts().OrderBy(o => r.Next(0, 10)).Take(filesToRemove).ToArray(); foreach (var part in parts) { package.DeletePart(part.Uri); } } } } } ================================================ FILE: source/Octodiff.Tests/PatchFixture.cs ================================================ using System; using System.IO; using System.Security.Cryptography; using NUnit.Framework; using Octodiff.Tests.Util; namespace Octodiff.Tests { [TestFixture] public class PatchFixture : CommandLineFixture { [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void PatchingShouldResultInPerfectCopy(string name, int numberOfFiles, OctodiffAppVariant octodiff) { var newName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2.zip"); var copyName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2_out.zip"); PackageGenerator.GeneratePackage(name, numberOfFiles); PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); Run("signature " + name + " " + name + ".sig", octodiff); Run("delta " + name + ".sig " + newName + " " + name + ".delta", octodiff); Run("patch " + name + " " + name + ".delta" + " " + copyName, octodiff); Assert.That(ExitCode, Is.EqualTo(0)); Assert.That(Sha1(newName), Is.EqualTo(Sha1(copyName))); } [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] public void PatchVerificationShouldFailWhenFilesModified(string name, int numberOfFiles, OctodiffAppVariant octodiff) { var newBasis = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "1.zip"); var newName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2.zip"); var copyName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2_out.zip"); PackageGenerator.GeneratePackage(name, numberOfFiles); PackageGenerator.ModifyPackage(name, newBasis, numberOfFiles, (int)(0.5 * numberOfFiles)); PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); Run("signature " + name + " " + name + ".sig", octodiff); Run("delta " + name + ".sig " + newName + " " + name + ".delta", octodiff); Run("patch " + newBasis + " " + name + ".delta" + " " + copyName, octodiff); Assert.That(ExitCode, Is.EqualTo(2)); Assert.That(Output, Does.Contain("Error: Verification of the patched file failed")); } [Test] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] public void PatchVerificationCanBeSkipped(string name, int numberOfFiles, OctodiffAppVariant octodiff) { var newBasis = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "1.zip"); var newName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2.zip"); var copyName = Path.ChangeExtension(Path.Combine(TestContext.CurrentContext.TestDirectory, name), "2_out.zip"); PackageGenerator.GeneratePackage(name, numberOfFiles); PackageGenerator.ModifyPackage(name, newBasis, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); Run("signature " + name + " " + name + ".sig", octodiff); Run("delta " + name + ".sig " + newName + " " + name + ".delta", octodiff); Run("patch " + newBasis + " " + name + ".delta" + " " + copyName + " --skip-verification", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); Assert.That(Sha1(newName), Is.Not.EqualTo(Sha1(copyName))); } static string Sha1(string fileName) { using (var s = new FileStream(fileName, FileMode.Open)) { return BitConverter.ToString(SHA1.Create().ComputeHash(s)).Replace("-", ""); } } } } ================================================ FILE: source/Octodiff.Tests/Properties/AssemblyInfo.cs ================================================ using System.Reflection; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("Octodiff.Tests")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] [assembly: AssemblyProduct("Octodiff.Tests")] [assembly: AssemblyCopyright("Copyright © 2014")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("ec22e906-c7ec-4b1f-adf4-f2419c22b173")] // Version information for an assembly consists of the following four values: // // Major Version // Minor Version // Build Number // Revision // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] [assembly: AssemblyVersion("1.0.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] ================================================ FILE: source/Octodiff.Tests/SignatureFixture.cs ================================================ using System.Diagnostics; using System.IO; using NUnit.Framework; using Octodiff.Tests.Util; namespace Octodiff.Tests { [TestFixture] public class SignatureFixture : CommandLineFixture { [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void ShouldCreateSignature(string name, int numberOfFiles, OctodiffAppVariant octodiff) { PackageGenerator.GeneratePackage(name, numberOfFiles); Run("signature " + name + " " + name + ".sig", octodiff); Assert.That(ExitCode, Is.EqualTo(0)); var basisSize = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, name)).Length; var signatureSize = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig").Length; var signatureSizePercentageOfBasis = signatureSize/(double) basisSize; Trace.WriteLine(string.Format("Basis size: {0:n0}", basisSize)); Trace.WriteLine(string.Format("Signature size: {0:n0}", signatureSize)); Trace.WriteLine(string.Format("Signature ratio: {0:n3}", signatureSizePercentageOfBasis)); Assert.That(0.006 <= signatureSizePercentageOfBasis && signatureSizePercentageOfBasis <= 0.014, Is.True); } [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void ShouldCreateDifferentSignaturesBasedOnChunkSize(string name, int numberOfFiles, OctodiffAppVariant octodiff) { PackageGenerator.GeneratePackage(name, numberOfFiles); Run("signature " + name + " " + name + ".sig.1 --chunk-size=128", octodiff); Run("signature " + name + " " + name + ".sig.2 --chunk-size=256", octodiff); Run("signature " + name + " " + name + ".sig.3 --chunk-size=1024", octodiff); Run("signature " + name + " " + name + ".sig.4 --chunk-size=2048", octodiff); Run("signature " + name + " " + name + ".sig.5 --chunk-size=31744", octodiff); Assert.That(Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.1") > Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.2")); Assert.That(Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.2") > Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.3")); Assert.That(Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.3") > Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.4")); Assert.That(Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.4") > Length(Path.Combine(TestContext.CurrentContext.TestDirectory, name) + ".sig.5")); } static long Length(string fileName) { return new FileInfo(fileName).Length; } } } ================================================ FILE: source/Octodiff.Tests/Timings.cs ================================================ using System; using System.Diagnostics; using System.IO; using NUnit.Framework; using Octodiff.Tests.Util; namespace Octodiff.Tests { [TestFixture] public class TimingsFixture : CommandLineFixture { [Test] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Sync)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Sync)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Sync)] [TestCase("SmallPackage1mb.zip", 10, OctodiffAppVariant.Async)] [TestCase("SmallPackage10mb.zip", 100, OctodiffAppVariant.Async)] [TestCase("SmallPackage100mb.zip", 1000, OctodiffAppVariant.Async)] public void ExecuteWithTimings(string name, int numberOfFiles, OctodiffAppVariant octodiff) { var newName = Path.ChangeExtension(name, "2.zip"); var copyName = Path.ChangeExtension(name, "2_out.zip"); Time("Package creation", () => PackageGenerator.GeneratePackage(name, numberOfFiles)); Time("Package modification", () => PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles))); Time("Signature creation", () => Run("signature " + name + " " + name + ".sig", octodiff)); Time("Delta creation", () => Run("delta " + name + ".sig " + newName + " " + name + ".delta", octodiff)); Time("Patch application", () => Run("patch " + name + " " + name + ".delta" + " " + copyName, octodiff)); Time("Patch application (no verify)", () => Run("patch " + name + " " + name + ".delta" + " " + copyName + " --skip-verification", octodiff)); } static void Time(string task, Action callback) { var watch = Stopwatch.StartNew(); callback(); Trace.WriteLine(task.PadRight(30, ' ') + ": " + watch.ElapsedMilliseconds + "ms"); } } } ================================================ FILE: source/Octodiff.Tests/Util/CommandLineFixture.cs ================================================ using System; using System.Diagnostics; using System.Text; using NUnit.Framework; using OctodiffAsync; using Octopus.Platform.Util; namespace Octodiff.Tests.Util { public enum OctodiffAppVariant { Sync, Async } public abstract class CommandLineFixture { protected string StdErr { get; private set; } protected string StdOut { get; private set; } protected string Output { get; private set; } protected int ExitCode { get; set; } public void Run(string args, OctodiffAppVariant octodiff) { var stdErrBuilder = new StringBuilder(); var stdOutBuilder = new StringBuilder(); var outputBuilder = new StringBuilder(); var path = octodiff == OctodiffAppVariant.Sync ? new Uri(typeof (OctodiffProgram).Assembly.CodeBase).LocalPath : new Uri(typeof(OctodiffAsyncProgram).Assembly.CodeBase).LocalPath; var exit = SilentProcessRunner.ExecuteCommand(path, args, TestContext.CurrentContext.TestDirectory, output => { stdOutBuilder.AppendLine(output); outputBuilder.AppendLine(output); Trace.WriteLine(output); }, output => { stdErrBuilder.AppendLine(output); outputBuilder.AppendLine(output); Trace.WriteLine(output); }); StdErr = stdErrBuilder.ToString(); StdOut = stdOutBuilder.ToString(); Output = outputBuilder.ToString(); ExitCode = exit; } } } ================================================ FILE: source/Octodiff.Tests/Util/SilentProcessRunner.cs ================================================ using System; using System.Diagnostics; using System.Runtime.InteropServices; using System.Text; using System.Threading; namespace Octopus.Platform.Util { public static class SilentProcessRunner { // ReSharper disable once InconsistentNaming private const int CP_OEMCP = 1; private static readonly Encoding oemEncoding; static SilentProcessRunner() { try { CPINFOEX info; if (GetCPInfoEx(CP_OEMCP, 0, out info)) { oemEncoding = Encoding.GetEncoding(info.CodePage); } else { oemEncoding = Encoding.GetEncoding(850); } } catch (Exception ) { oemEncoding = Encoding.UTF8; } } public static int ExecuteCommand(string executable, string arguments, string workingDirectory, Action output, Action error) { try { using (var process = new Process()) { process.StartInfo.FileName = executable; process.StartInfo.Arguments = arguments; process.StartInfo.WorkingDirectory = workingDirectory; process.StartInfo.UseShellExecute = false; process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.StandardOutputEncoding = oemEncoding; process.StartInfo.StandardErrorEncoding = oemEncoding; using (var outputWaitHandle = new AutoResetEvent(false)) using (var errorWaitHandle = new AutoResetEvent(false)) { process.OutputDataReceived += (sender, e) => { if (e.Data == null) { outputWaitHandle.Set(); } else { output(e.Data); } }; process.ErrorDataReceived += (sender, e) => { if (e.Data == null) { errorWaitHandle.Set(); } else { error(e.Data); } }; process.Start(); process.BeginOutputReadLine(); process.BeginErrorReadLine(); process.WaitForExit(); outputWaitHandle.WaitOne(); errorWaitHandle.WaitOne(); return process.ExitCode; } } } catch (Exception ex) { throw new Exception(string.Format("Error when attempting to execute {0}: {1}", executable, ex.Message), ex); } } [DllImport("kernel32.dll", SetLastError = true)] private static extern bool GetCPInfoEx([MarshalAs(UnmanagedType.U4)] int CodePage, [MarshalAs(UnmanagedType.U4)] int dwFlags, out CPINFOEX lpCPInfoEx); private const int MAX_DEFAULTCHAR = 2; private const int MAX_LEADBYTES = 12; private const int MAX_PATH = 260; [StructLayout(LayoutKind.Sequential)] private struct CPINFOEX { [MarshalAs(UnmanagedType.U4)] public int MaxCharSize; [MarshalAs(UnmanagedType.ByValArray, SizeConst = MAX_DEFAULTCHAR)] public byte[] DefaultChar; [MarshalAs(UnmanagedType.ByValArray, SizeConst = MAX_LEADBYTES)] public byte[] LeadBytes; public char UnicodeDefaultChar; [MarshalAs(UnmanagedType.U4)] public int CodePage; [MarshalAs(UnmanagedType.ByValTStr, SizeConst = MAX_PATH)] public string CodePageName; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/DeltaCommand.cs ================================================ using System; using System.Collections.Generic; using System.IO; using FastRsync.Core; using FastRsync.Delta; using FastRsync.Diagnostics; using FastRsync.Signature; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync.CommandLine { [Command("delta", Description = "Given a signature file and a new file, creates a delta file", Usage = " []")] class DeltaCommand : ICommand { private readonly List> configuration = new List>(); private readonly OptionSet options; private string newFilePath; private string signatureFilePath; private string deltaFilePath; public DeltaCommand() { options = new OptionSet(); options.Positional("signature-file", "The file containing the signature from the basis file.", v => signatureFilePath = v); options.Positional("new-file", "The file to create the delta from.", v => newFilePath = v); options.Positional("delta-file", "The file to write the delta to.", v => deltaFilePath = v); options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReport = new ConsoleProgressReporter())); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(signatureFilePath)) throw new OptionException("No signature file was specified", "new-file"); if (string.IsNullOrWhiteSpace(newFilePath)) throw new OptionException("No new file was specified", "new-file"); newFilePath = Path.GetFullPath(newFilePath); signatureFilePath = Path.GetFullPath(signatureFilePath); var delta = new DeltaBuilder(); foreach (var config in configuration) config(delta); if (!File.Exists(signatureFilePath)) { throw new FileNotFoundException("File not found: " + signatureFilePath, signatureFilePath); } if (!File.Exists(newFilePath)) { throw new FileNotFoundException("File not found: " + newFilePath, newFilePath); } if (string.IsNullOrWhiteSpace(deltaFilePath)) { deltaFilePath = newFilePath + ".octodelta"; } else { deltaFilePath = Path.GetFullPath(deltaFilePath); var directory = Path.GetDirectoryName(deltaFilePath); if (directory != null && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } } using (var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { delta.BuildDeltaAsync(newFileStream, new SignatureReader(signatureStream, delta.ProgressReport), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).GetAwaiter().GetResult(); } return 0; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/ExplainDeltaCommand.cs ================================================ using System; using System.IO; using System.Linq; using System.Threading.Tasks; using FastRsync.Delta; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync.CommandLine { [Command("explain-delta", Description = "Prints instructions from a delta file; useful when debugging.", Usage = "")] class ExplainDeltaCommand : ICommand { private readonly OptionSet options; private string deltaFilePath; public ExplainDeltaCommand() { options = new OptionSet(); options.Positional("delta-file", "The file to read the delta from.", v => deltaFilePath = v); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(deltaFilePath)) throw new OptionException("No delta file was specified", "delta-file"); deltaFilePath = Path.GetFullPath(deltaFilePath); if (!File.Exists(deltaFilePath)) { throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); } using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { var reader = new BinaryDeltaReader(deltaStream, null); reader.ApplyAsync(data => { if (data.Length > 20) { Console.WriteLine("Data: ({0} bytes): {1}...", data.Length, BitConverter.ToString(data.Take(20).ToArray())); } else { Console.WriteLine("Data: ({0} bytes): {1}", data.Length, BitConverter.ToString(data.ToArray())); } return Task.CompletedTask; }, (start, offset) => { Console.WriteLine("Copy: {0:X} to {1:X}", start, offset); return Task.CompletedTask; }).GetAwaiter().GetResult(); } return 0; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/HelpCommand.cs ================================================ using System; using System.IO; using System.Linq; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync.CommandLine { [Command("help", "?", "h", Description = "Prints this help text")] class HelpCommand : ICommand { readonly ICommandLocator commands = new CommandLocator(); public void GetHelp(TextWriter writer) { } public int Execute(string[] commandLineArguments) { var executable = Path.GetFileNameWithoutExtension(new Uri(typeof(HelpCommand).Assembly.CodeBase).LocalPath); var commandName = commandLineArguments.FirstOrDefault(); if (string.IsNullOrEmpty(commandName)) { PrintGeneralHelp(executable); } else { var commandMeta = commands.Find(commandName); if (commandMeta == null) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Command '{0}' is not supported", commandName); Console.ResetColor(); PrintGeneralHelp(executable); } else { var command = commands.Create(commandMeta); PrintCommandHelp(executable, command, commandMeta, commandName); } } return 0; } void PrintCommandHelp(string executable, ICommand command, ICommandMetadata commandMetadata, string commandName) { Console.ResetColor(); Console.Write("Usage: "); Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(executable + " " + commandName + (!string.IsNullOrWhiteSpace(commandMetadata.Usage) ? " " + commandMetadata.Usage : "") + " []"); Console.ResetColor(); Console.WriteLine(); command.GetHelp(Console.Out); Console.WriteLine(); } void PrintGeneralHelp(string executable) { Console.ResetColor(); Console.Write("Usage: "); Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(executable + " "); Console.ResetColor(); Console.WriteLine(); Console.WriteLine("Where is one of: "); Console.WriteLine(); foreach (var possible in commands.List().OrderBy(x => x.Name)) { Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(" " + possible.Name.PadRight(15, ' ')); Console.ResetColor(); Console.WriteLine(" " + possible.Description); } Console.WriteLine(); Console.Write("Or use "); Console.ForegroundColor = ConsoleColor.White; Console.Write(executable + " help "); Console.ResetColor(); Console.WriteLine(" for more details."); } } } ================================================ FILE: source/OctodiffAsync/CommandLine/PatchCommand.cs ================================================ using System; using System.IO; using FastRsync.Delta; using FastRsync.Diagnostics; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync.CommandLine { [Command("patch", Description = "Given a basis file, and a delta, produces the new file", Usage = " ")] class PatchCommand : ICommand { private readonly OptionSet options; private IProgress progressReport; private string basisFilePath; private string deltaFilePath; private string newFilePath; private bool skipHashCheck; public PatchCommand() { options = new OptionSet(); options.Positional("basis-file", "The file that the delta was created for.", v => basisFilePath = v); options.Positional("delta-file", "The delta to apply to the basis file", v => deltaFilePath = v); options.Positional("new-file", "The file to write the result to.", v => newFilePath = v); options.Add("progress", "Whether progress should be written to stdout", v => progressReport = new ConsoleProgressReporter()); options.Add("skip-verification", "Skip checking whether the basis file is the same as the file used to produce the signature that created the delta.", v => skipHashCheck = true); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(basisFilePath)) throw new OptionException("No basis file was specified", "basis-file"); if (string.IsNullOrWhiteSpace(deltaFilePath)) throw new OptionException("No delta file was specified", "delta-file"); if (string.IsNullOrWhiteSpace(newFilePath)) throw new OptionException("No new file was specified", "new-file"); basisFilePath = Path.GetFullPath(basisFilePath); deltaFilePath = Path.GetFullPath(deltaFilePath); newFilePath = Path.GetFullPath(newFilePath); if (!File.Exists(basisFilePath)) throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); if (!File.Exists(deltaFilePath)) throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); var directory = Path.GetDirectoryName(newFilePath); if (directory != null && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var delta = new DeltaApplier { SkipHashCheck = skipHashCheck }; using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var newFileStream = new FileStream(newFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) { delta.ApplyAsync(basisStream, new BinaryDeltaReader(deltaStream, progressReport), newFileStream).GetAwaiter().GetResult(); } return 0; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/SignatureCommand.cs ================================================ using System; using System.Collections.Generic; using System.IO; using FastRsync.Diagnostics; using FastRsync.Signature; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync.CommandLine { [Command("signature", "sig", Description = "Given a basis file, creates a signature file", Usage = " []")] class SignatureCommand : ICommand { private readonly List> configuration = new List>(); private readonly OptionSet options; private string basisFilePath; private string signatureFilePath; public SignatureCommand() { options = new OptionSet(); options.Positional("basis-file", "The file to read and create a signature from.", v => basisFilePath = v); options.Positional("signature-file", "The file to write the signature to.", v => signatureFilePath = v); options.Add("chunk-size=", string.Format("Maximum bytes per chunk. Defaults to {0}. Min of {1}, max of {2}.", SignatureBuilder.DefaultChunkSize, SignatureBuilder.MinimumChunkSize, SignatureBuilder.MaximumChunkSize), v => configuration.Add(builder => builder.ChunkSize = short.Parse(v))); options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReport = new ConsoleProgressReporter())); } public void GetHelp(TextWriter writer) { options.WriteOptionDescriptions(writer); } public int Execute(string[] commandLineArguments) { options.Parse(commandLineArguments); if (string.IsNullOrWhiteSpace(basisFilePath)) throw new OptionException("No basis file was specified", "basis-file"); basisFilePath = Path.GetFullPath(basisFilePath); var signatureBuilder = new SignatureBuilder(); foreach (var config in configuration) config(signatureBuilder); if (!File.Exists(basisFilePath)) { throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); } if (string.IsNullOrWhiteSpace(signatureFilePath)) { signatureFilePath = basisFilePath + ".octosig"; } else { signatureFilePath = Path.GetFullPath(signatureFilePath); var sigDirectory = Path.GetDirectoryName(signatureFilePath); if (sigDirectory != null && !Directory.Exists(sigDirectory)) { Directory.CreateDirectory(sigDirectory); } } using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { signatureBuilder.BuildAsync(basisStream, new SignatureWriter(signatureStream)).GetAwaiter().GetResult(); } return 0; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/CommandAttribute.cs ================================================ using System; namespace OctodiffAsync.CommandLine.Support { [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] class CommandAttribute : Attribute, ICommandMetadata { public CommandAttribute(string name, params string[] aliases) { Name = name; Aliases = aliases; } public string Name { get; set; } public string[] Aliases { get; set; } public string Description { get; set; } public string Usage { get; set; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/CommandException.cs ================================================ using System; namespace OctodiffAsync.CommandLine.Support { class CommandException : Exception { public CommandException(string message) : base(message) { } } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/CommandLocator.cs ================================================ using System; using System.Linq; namespace OctodiffAsync.CommandLine.Support { class CommandLocator : ICommandLocator { public ICommandMetadata[] List() { return (from t in typeof (CommandLocator).Assembly.GetTypes() where typeof (ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata) t.GetCustomAttributes(typeof (CommandAttribute), true).FirstOrDefault() where attribute != null select attribute).ToArray(); } public ICommandMetadata Find(string name) { name = name.Trim().ToLowerInvariant(); return (from t in typeof (CommandLocator).Assembly.GetTypes() where typeof (ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata) t.GetCustomAttributes(typeof (CommandAttribute), true).FirstOrDefault() where attribute != null where attribute.Name == name || attribute.Aliases.Any(a => a == name) select attribute).FirstOrDefault(); } public ICommand Create(ICommandMetadata metadata) { var name = metadata.Name; var found = (from t in typeof(CommandLocator).Assembly.GetTypes() where typeof(ICommand).IsAssignableFrom(t) let attribute = (ICommandMetadata)t.GetCustomAttributes(typeof(CommandAttribute), true).FirstOrDefault() where attribute != null where attribute.Name == name || attribute.Aliases.Any(a => a == name) select t).FirstOrDefault(); return found == null ? null : (ICommand)Activator.CreateInstance(found); } } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/ICommand.cs ================================================ using System.IO; namespace OctodiffAsync.CommandLine.Support { interface ICommand { void GetHelp(TextWriter writer); int Execute(string[] commandLineArguments); } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/ICommandLocator.cs ================================================ namespace OctodiffAsync.CommandLine.Support { interface ICommandLocator { ICommandMetadata[] List(); ICommandMetadata Find(string name); ICommand Create(ICommandMetadata metadata); } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/ICommandMetadata.cs ================================================ namespace OctodiffAsync.CommandLine.Support { interface ICommandMetadata { string Name { get; } string[] Aliases { get; } string Description { get; } string Usage { get; set; } } } ================================================ FILE: source/OctodiffAsync/CommandLine/Support/NDesk.Options.cs ================================================ // // Options.cs // // Authors: // Jonathan Pryor // // Copyright (C) 2008 Novell (http://www.novell.com) // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // Compile With: // gmcs -debug+ -r:System.Core Options.cs -o:NDesk.Options.dll // gmcs -debug+ -d:LINQ -r:System.Core Options.cs -o:NDesk.Options.dll // // The LINQ version just changes the implementation of // OptionSet.Parse(IEnumerable), and confers no semantic changes. // // A Getopt::Long-inspired option parsing library for C#. // // NDesk.Options.OptionSet is built upon a key/value table, where the // key is a option format string and the value is a delegate that is // invoked when the format string is matched. // // Option format strings: // Regex-like BNF Grammar: // name: .+ // type: [=:] // sep: ( [^{}]+ | '{' .+ '}' )? // aliases: ( name type sep ) ( '|' name type sep )* // // Each '|'-delimited name is an alias for the associated action. If the // format string ends in a '=', it has a required value. If the format // string ends in a ':', it has an optional value. If neither '=' or ':' // is present, no value is supported. `=' or `:' need only be defined on one // alias, but if they are provided on more than one they must be consistent. // // Each alias portion may also end with a "key/value separator", which is used // to split option values if the option accepts > 1 value. If not specified, // it defaults to '=' and ':'. If specified, it can be any character except // '{' and '}' OR the *string* between '{' and '}'. If no separator should be // used (i.e. the separate values should be distinct arguments), then "{}" // should be used as the separator. // // Options are extracted either from the current option by looking for // the option name followed by an '=' or ':', or is taken from the // following option IFF: // - The current option does not contain a '=' or a ':' // - The current option requires a value (i.e. not a Option type of ':') // // The `name' used in the option format string does NOT include any leading // option indicator, such as '-', '--', or '/'. All three of these are // permitted/required on any named option. // // Option bundling is permitted so long as: // - '-' is used to start the option group // - all of the bundled options are a single character // - at most one of the bundled options accepts a value, and the value // provided starts from the next character to the end of the string. // // This allows specifying '-a -b -c' as '-abc', and specifying '-D name=value' // as '-Dname=value'. // // Option processing is disabled by specifying "--". All options after "--" // are returned by OptionSet.Parse() unchanged and unprocessed. // // Unprocessed options are returned from OptionSet.Parse(). // // Examples: // int verbose = 0; // OptionSet p = new OptionSet () // .Add ("v", v => ++verbose) // .Add ("name=|value=", v => Console.WriteLine (v)); // p.Parse (new string[]{"-v", "--v", "/v", "-name=A", "/name", "B", "extra"}); // // The above would parse the argument string array, and would invoke the // lambda expression three times, setting `verbose' to 3 when complete. // It would also print out "A" and "B" to standard output. // The returned array would contain the string "extra". // // C# 3.0 collection initializers are supported and encouraged: // var p = new OptionSet () { // { "h|?|help", v => ShowHelp () }, // }; // // System.ComponentModel.TypeConverter is also supported, allowing the use of // custom data types in the callback type; TypeConverter.ConvertFromString() // is used to convert the value option to an instance of the specified // type: // // var p = new OptionSet () { // { "foo=", (Foo f) => Console.WriteLine (f.ToString ()) }, // }; // // Random other tidbits: // - Boolean options (those w/o '=' or ':' in the option format string) // are explicitly enabled if they are followed with '+', and explicitly // disabled if they are followed with '-': // string a = null; // var p = new OptionSet () { // { "a", s => a = s }, // }; // p.Parse (new string[]{"-a"}); // sets v != null // p.Parse (new string[]{"-a+"}); // sets v != null // p.Parse (new string[]{"-a-"}); // sets v == null // using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.ComponentModel; using System.IO; using System.Linq; using System.Runtime.Serialization; using System.Security.Permissions; using System.Text; using System.Text.RegularExpressions; #if LINQ using System.Linq; #endif #if TEST using NDesk.Options; #endif #if NDESK_OPTIONS namespace NDesk.Options #else namespace OctodiffAsync.CommandLine.Support #endif { class OptionValueCollection : IList, IList { List values = new List(); OptionContext c; internal OptionValueCollection(OptionContext c) { this.c = c; } #region ICollection void ICollection.CopyTo(Array array, int index) { (values as ICollection).CopyTo(array, index); } bool ICollection.IsSynchronized { get { return (values as ICollection).IsSynchronized; } } object ICollection.SyncRoot { get { return (values as ICollection).SyncRoot; } } #endregion #region ICollection public void Add(string item) { values.Add(item); } public void Clear() { values.Clear(); } public bool Contains(string item) { return values.Contains(item); } public void CopyTo(string[] array, int arrayIndex) { values.CopyTo(array, arrayIndex); } public bool Remove(string item) { return values.Remove(item); } public int Count { get { return values.Count; } } public bool IsReadOnly { get { return false; } } #endregion #region IEnumerable IEnumerator IEnumerable.GetEnumerator() { return values.GetEnumerator(); } #endregion #region IEnumerable public IEnumerator GetEnumerator() { return values.GetEnumerator(); } #endregion #region IList int IList.Add(object value) { return (values as IList).Add(value); } bool IList.Contains(object value) { return (values as IList).Contains(value); } int IList.IndexOf(object value) { return (values as IList).IndexOf(value); } void IList.Insert(int index, object value) { (values as IList).Insert(index, value); } void IList.Remove(object value) { (values as IList).Remove(value); } void IList.RemoveAt(int index) { (values as IList).RemoveAt(index); } bool IList.IsFixedSize { get { return false; } } object IList.this[int index] { get { return this[index]; } set { (values as IList)[index] = value; } } #endregion #region IList public int IndexOf(string item) { return values.IndexOf(item); } public void Insert(int index, string item) { values.Insert(index, item); } public void RemoveAt(int index) { values.RemoveAt(index); } private void AssertValid(int index) { if (c.Option == null) throw new InvalidOperationException("OptionContext.Option is null."); if (index >= c.Option.MaxValueCount) throw new ArgumentOutOfRangeException("index"); if (c.Option.OptionValueType == OptionValueType.Required && index >= values.Count) throw new OptionException(string.Format( c.OptionSet.MessageLocalizer("Missing required value for option '{0}'."), c.OptionName), c.OptionName); } public string this[int index] { get { AssertValid(index); return index >= values.Count ? null : values[index]; } set { values[index] = value; } } #endregion public List ToList() { return new List(values); } public string[] ToArray() { return values.ToArray(); } public override string ToString() { return string.Join(", ", values.ToArray()); } } class OptionContext { private Option option; private string name; private int index; private OptionSet set; private OptionValueCollection c; public OptionContext(OptionSet set) { this.set = set; this.c = new OptionValueCollection(this); } public Option Option { get { return option; } set { option = value; } } public string OptionName { get { return name; } set { name = value; } } public int OptionIndex { get { return index; } set { index = value; } } public OptionSet OptionSet { get { return set; } } public OptionValueCollection OptionValues { get { return c; } } } enum OptionValueType { None, Optional, Required, } abstract class Option { string prototype, description; string[] names; OptionValueType type; int count; string[] separators; protected Option(string prototype, string description) : this(prototype, description, 1) { } protected Option(string prototype, string description, int maxValueCount) { if (prototype == null) throw new ArgumentNullException("prototype"); if (prototype.Length == 0) throw new ArgumentException("Cannot be the empty string.", "prototype"); if (maxValueCount < 0) throw new ArgumentOutOfRangeException("maxValueCount"); this.prototype = prototype; this.names = prototype.Split('|'); this.description = description; this.count = maxValueCount; this.type = ParsePrototype(); if (this.count == 0 && type != OptionValueType.None) throw new ArgumentException( "Cannot provide maxValueCount of 0 for OptionValueType.Required or " + "OptionValueType.Optional.", "maxValueCount"); if (this.type == OptionValueType.None && maxValueCount > 1) throw new ArgumentException( string.Format("Cannot provide maxValueCount of {0} for OptionValueType.None.", maxValueCount), "maxValueCount"); if (Array.IndexOf(names, "<>") >= 0 && ((names.Length == 1 && this.type != OptionValueType.None) || (names.Length > 1 && this.MaxValueCount > 1))) throw new ArgumentException( "The default option handler '<>' cannot require values.", "prototype"); } public string Prototype { get { return prototype; } } public string Description { get { return description; } } public OptionValueType OptionValueType { get { return type; } } public int MaxValueCount { get { return count; } } public string[] GetNames() { return (string[])names.Clone(); } public string[] GetValueSeparators() { if (separators == null) return new string[0]; return (string[])separators.Clone(); } protected static T Parse(string value, OptionContext c) { Type tt = typeof(T); bool nullable = tt.IsValueType && tt.IsGenericType && !tt.IsGenericTypeDefinition && tt.GetGenericTypeDefinition() == typeof(Nullable<>); Type targetType = nullable ? tt.GetGenericArguments()[0] : typeof(T); TypeConverter conv = TypeDescriptor.GetConverter(targetType); T t = default(T); try { if (value != null) t = (T)conv.ConvertFromString(value); } catch (Exception e) { throw new OptionException( string.Format( c.OptionSet.MessageLocalizer("Could not convert string `{0}' to type {1} for option `{2}'."), value, targetType.Name, c.OptionName), c.OptionName, e); } return t; } internal string[] Names { get { return names; } } internal string[] ValueSeparators { get { return separators; } } static readonly char[] NameTerminator = new char[] { '=', ':' }; private OptionValueType ParsePrototype() { char type = '\0'; List seps = new List(); for (int i = 0; i < names.Length; ++i) { string name = names[i]; if (name.Length == 0) throw new ArgumentException("Empty option names are not supported.", "prototype"); int end = name.IndexOfAny(NameTerminator); if (end == -1) continue; names[i] = name.Substring(0, end); if (type == '\0' || type == name[end]) type = name[end]; else throw new ArgumentException( string.Format("Conflicting option types: '{0}' vs. '{1}'.", type, name[end]), "prototype"); AddSeparators(name, end, seps); } if (type == '\0') return OptionValueType.None; if (count <= 1 && seps.Count != 0) throw new ArgumentException( string.Format("Cannot provide key/value separators for Options taking {0} value(s).", count), "prototype"); if (count > 1) { if (seps.Count == 0) this.separators = new string[] { ":", "=" }; else if (seps.Count == 1 && seps[0].Length == 0) this.separators = null; else this.separators = seps.ToArray(); } return type == '=' ? OptionValueType.Required : OptionValueType.Optional; } private static void AddSeparators(string name, int end, ICollection seps) { int start = -1; for (int i = end + 1; i < name.Length; ++i) { switch (name[i]) { case '{': if (start != -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); start = i + 1; break; case '}': if (start == -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); seps.Add(name.Substring(start, i - start)); start = -1; break; default: if (start == -1) seps.Add(name[i].ToString()); break; } } if (start != -1) throw new ArgumentException( string.Format("Ill-formed name/value separator found in \"{0}\".", name), "prototype"); } public void Invoke(OptionContext c) { OnParseComplete(c); c.OptionName = null; c.Option = null; c.OptionValues.Clear(); } protected abstract void OnParseComplete(OptionContext c); public override string ToString() { return Prototype; } } [Serializable] class OptionException : Exception { private string option; public OptionException() { } public OptionException(string message, string optionName) : base(message) { this.option = optionName; } public OptionException(string message, string optionName, Exception innerException) : base(message, innerException) { this.option = optionName; } protected OptionException(SerializationInfo info, StreamingContext context) : base(info, context) { this.option = info.GetString("OptionName"); } public string OptionName { get { return this.option; } } [SecurityPermission(SecurityAction.LinkDemand, SerializationFormatter = true)] public override void GetObjectData(SerializationInfo info, StreamingContext context) { base.GetObjectData(info, context); info.AddValue("OptionName", option); } } delegate void OptionAction(TKey key, TValue value); class OptionSet : KeyedCollection { readonly List>> positionalParameters = new List>>(); public OptionSet() : this(delegate(string f) { return f; }) { } public OptionSet(Converter localizer) { this.localizer = localizer; } Converter localizer; public Converter MessageLocalizer { get { return localizer; } } protected override string GetKeyForItem(Option item) { if (item == null) throw new ArgumentNullException("option"); if (item.Names != null && item.Names.Length > 0) return item.Names[0]; // This should never happen, as it's invalid for Option to be // constructed w/o any names. throw new InvalidOperationException("Option has no names!"); } [Obsolete("Use KeyedCollection.this[string]")] protected Option GetOptionForName(string option) { if (option == null) throw new ArgumentNullException("option"); try { return base[option]; } catch (KeyNotFoundException) { return null; } } protected override void InsertItem(int index, Option item) { base.InsertItem(index, item); AddImpl(item); } protected override void RemoveItem(int index) { base.RemoveItem(index); Option p = Items[index]; // KeyedCollection.RemoveItem() handles the 0th item for (int i = 1; i < p.Names.Length; ++i) { Dictionary.Remove(p.Names[i]); } } protected override void SetItem(int index, Option item) { base.SetItem(index, item); RemoveItem(index); AddImpl(item); } private void AddImpl(Option option) { if (option == null) throw new ArgumentNullException("option"); List added = new List(option.Names.Length); try { // KeyedCollection.InsertItem/SetItem handle the 0th name. for (int i = 1; i < option.Names.Length; ++i) { Dictionary.Add(option.Names[i], option); added.Add(option.Names[i]); } } catch (Exception) { foreach (string name in added) Dictionary.Remove(name); throw; } } public new OptionSet Add(Option option) { base.Add(option); return this; } public void Positional(string name, string description, Action valueCallback) { positionalParameters.Add(Tuple.Create(name, description, valueCallback)); } sealed class ActionOption : Option { Action action; public ActionOption(string prototype, string description, int count, Action action) : base(prototype, description, count) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action(c.OptionValues); } } public OptionSet Add(string prototype, Action action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, Action action) { if (action == null) throw new ArgumentNullException("action"); Option p = new ActionOption(prototype, description, 1, delegate(OptionValueCollection v) { action(v[0]); }); base.Add(p); return this; } public OptionSet Add(string prototype, OptionAction action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, OptionAction action) { if (action == null) throw new ArgumentNullException("action"); Option p = new ActionOption(prototype, description, 2, delegate(OptionValueCollection v) { action(v[0], v[1]); }); base.Add(p); return this; } sealed class ActionOption : Option { Action action; public ActionOption(string prototype, string description, Action action) : base(prototype, description, 1) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action(Parse(c.OptionValues[0], c)); } } sealed class ActionOption : Option { OptionAction action; public ActionOption(string prototype, string description, OptionAction action) : base(prototype, description, 2) { if (action == null) throw new ArgumentNullException("action"); this.action = action; } protected override void OnParseComplete(OptionContext c) { action( Parse(c.OptionValues[0], c), Parse(c.OptionValues[1], c)); } } public OptionSet Add(string prototype, Action action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, Action action) { return Add(new ActionOption(prototype, description, action)); } public OptionSet Add(string prototype, OptionAction action) { return Add(prototype, null, action); } public OptionSet Add(string prototype, string description, OptionAction action) { return Add(new ActionOption(prototype, description, action)); } protected virtual OptionContext CreateOptionContext() { return new OptionContext(this); } #if LINQ public List Parse (IEnumerable arguments) { bool process = true; OptionContext c = CreateOptionContext (); c.OptionIndex = -1; var def = GetOptionForName ("<>"); var unprocessed = from argument in arguments where ++c.OptionIndex >= 0 && (process || def != null) ? process ? argument == "--" ? (process = false) : !Parse (argument, c) ? def != null ? Unprocessed (null, def, c, argument) : true : false : def != null ? Unprocessed (null, def, c, argument) : true : true select argument; List r = unprocessed.ToList (); if (c.Option != null) c.Option.Invoke (c); return r; } #else public List Parse(IEnumerable arguments) { OptionContext c = CreateOptionContext(); c.OptionIndex = -1; bool process = true; List unprocessed = new List(); Option def = Contains("<>") ? this["<>"] : null; foreach (string argument in arguments) { ++c.OptionIndex; if (argument == "--") { process = false; continue; } if (!process) { Unprocessed(unprocessed, def, c, argument); continue; } if (!Parse(argument, c)) Unprocessed(unprocessed, def, c, argument); } if (c.Option != null) c.Option.Invoke(c); for (var i = 0; i < positionalParameters.Count; i++) { if (i < unprocessed.Count) { positionalParameters[i].Item3(unprocessed[i]); } } return unprocessed; } #endif private static bool Unprocessed(ICollection extra, Option def, OptionContext c, string argument) { if (def == null) { extra.Add(argument); return false; } c.OptionValues.Add(argument); c.Option = def; c.Option.Invoke(c); return false; } private readonly Regex ValueOption = new Regex( @"^(?--|-|/)(?[^:=]+)((?[:=])(?.*))?$"); protected bool GetOptionParts(string argument, out string flag, out string name, out string sep, out string value) { if (argument == null) throw new ArgumentNullException("argument"); flag = name = sep = value = null; Match m = ValueOption.Match(argument); if (!m.Success) { return false; } flag = m.Groups["flag"].Value; name = m.Groups["name"].Value; if (m.Groups["sep"].Success && m.Groups["value"].Success) { sep = m.Groups["sep"].Value; value = m.Groups["value"].Value; } return true; } protected virtual bool Parse(string argument, OptionContext c) { if (c.Option != null) { ParseValue(argument, c); return true; } string f, n, s, v; if (!GetOptionParts(argument, out f, out n, out s, out v)) return false; Option p; if (Contains(n)) { p = this[n]; c.OptionName = f + n; c.Option = p; switch (p.OptionValueType) { case OptionValueType.None: c.OptionValues.Add(n); c.Option.Invoke(c); break; case OptionValueType.Optional: case OptionValueType.Required: ParseValue(v, c); break; } return true; } // no match; is it a bool option? if (ParseBool(argument, n, c)) return true; // is it a bundled option? if (ParseBundledValue(f, string.Concat(n + s + v), c)) return true; return false; } private void ParseValue(string option, OptionContext c) { if (option != null) foreach (string o in c.Option.ValueSeparators != null ? option.Split(c.Option.ValueSeparators, StringSplitOptions.None) : new string[] { option }) { c.OptionValues.Add(o); } if (c.OptionValues.Count == c.Option.MaxValueCount || c.Option.OptionValueType == OptionValueType.Optional) c.Option.Invoke(c); else if (c.OptionValues.Count > c.Option.MaxValueCount) { throw new OptionException(localizer(string.Format( "Error: Found {0} option values when expecting {1}.", c.OptionValues.Count, c.Option.MaxValueCount)), c.OptionName); } } private bool ParseBool(string option, string n, OptionContext c) { Option p; string rn; if (n.Length >= 1 && (n[n.Length - 1] == '+' || n[n.Length - 1] == '-') && Contains((rn = n.Substring(0, n.Length - 1)))) { p = this[rn]; string v = n[n.Length - 1] == '+' ? option : null; c.OptionName = option; c.Option = p; c.OptionValues.Add(v); p.Invoke(c); return true; } return false; } private bool ParseBundledValue(string f, string n, OptionContext c) { if (f != "-") return false; for (int i = 0; i < n.Length; ++i) { Option p; string opt = f + n[i].ToString(); string rn = n[i].ToString(); if (!Contains(rn)) { if (i == 0) return false; throw new OptionException(string.Format(localizer( "Cannot bundle unregistered option '{0}'."), opt), opt); } p = this[rn]; switch (p.OptionValueType) { case OptionValueType.None: Invoke(c, opt, n, p); break; case OptionValueType.Optional: case OptionValueType.Required: { string v = n.Substring(i + 1); c.Option = p; c.OptionName = opt; ParseValue(v.Length != 0 ? v : null, c); return true; } default: throw new InvalidOperationException("Unknown OptionValueType: " + p.OptionValueType); } } return true; } private static void Invoke(OptionContext c, string name, string value, Option option) { c.OptionName = name; c.Option = option; c.OptionValues.Add(value); option.Invoke(c); } private const int OptionWidth = 29; public List GetPositionals() { return positionalParameters.Select(s => s.Item1).ToList(); } public void WriteOptionDescriptions(TextWriter o) { if (positionalParameters.Count > 0) { o.WriteLine("Arguments:"); o.WriteLine(); } foreach (var positional in positionalParameters) { var prototype = " " + positional.Item1; o.Write(prototype); var written = prototype.Length; if (written < OptionWidth) o.Write(new string(' ', OptionWidth - written)); else { o.WriteLine(); o.Write(new string(' ', OptionWidth)); } bool indent = false; string prefix = new string(' ', OptionWidth); foreach (string line in GetLines(positional.Item2)) { if (indent) o.Write(prefix); o.WriteLine(line); indent = true; } } if (this.Count > 0) { if (positionalParameters.Count > 0) { o.WriteLine(); } o.WriteLine("Options:"); o.WriteLine(); } foreach (Option p in this) { int written = 0; if (!WriteOptionPrototype(o, p, ref written)) continue; if (written < OptionWidth) o.Write(new string(' ', OptionWidth - written)); else { o.WriteLine(); o.Write(new string(' ', OptionWidth)); } bool indent = false; string prefix = new string(' ', OptionWidth); foreach (string line in GetLines(localizer(GetDescription(p.Description)))) { if (indent) o.Write(prefix); o.WriteLine(line); indent = true; } } } bool WriteOptionPrototype(TextWriter o, Option p, ref int written) { string[] names = p.Names; int i = GetNextOptionIndex(names, 0); if (i == names.Length) return false; if (names[i].Length == 1) { Write(o, ref written, " -"); Write(o, ref written, names[0]); } else { Write(o, ref written, " --"); Write(o, ref written, names[0]); } for (i = GetNextOptionIndex(names, i + 1); i < names.Length; i = GetNextOptionIndex(names, i + 1)) { Write(o, ref written, ", "); Write(o, ref written, names[i].Length == 1 ? "-" : "--"); Write(o, ref written, names[i]); } if (p.OptionValueType == OptionValueType.Optional || p.OptionValueType == OptionValueType.Required) { if (p.OptionValueType == OptionValueType.Optional) { Write(o, ref written, localizer("[")); } Write(o, ref written, localizer("=" + GetArgumentName(0, p.MaxValueCount, p.Description))); string sep = p.ValueSeparators != null && p.ValueSeparators.Length > 0 ? p.ValueSeparators[0] : " "; for (int c = 1; c < p.MaxValueCount; ++c) { Write(o, ref written, localizer(sep + GetArgumentName(c, p.MaxValueCount, p.Description))); } if (p.OptionValueType == OptionValueType.Optional) { Write(o, ref written, localizer("]")); } } return true; } static int GetNextOptionIndex(string[] names, int i) { while (i < names.Length && names[i] == "<>") { ++i; } return i; } static void Write(TextWriter o, ref int n, string s) { n += s.Length; o.Write(s); } private static string GetArgumentName(int index, int maxIndex, string description) { if (description == null) return maxIndex == 1 ? "VALUE" : "VALUE" + (index + 1); string[] nameStart; if (maxIndex == 1) nameStart = new string[] { "{0:", "{" }; else nameStart = new string[] { "{" + index + ":" }; for (int i = 0; i < nameStart.Length; ++i) { int start, j = 0; do { start = description.IndexOf(nameStart[i], j); } while (start >= 0 && j != 0 ? description[j++ - 1] == '{' : false); if (start == -1) continue; int end = description.IndexOf("}", start); if (end == -1) continue; return description.Substring(start + nameStart[i].Length, end - start - nameStart[i].Length); } return maxIndex == 1 ? "VALUE" : "VALUE" + (index + 1); } private static string GetDescription(string description) { if (description == null) return string.Empty; StringBuilder sb = new StringBuilder(description.Length); int start = -1; for (int i = 0; i < description.Length; ++i) { switch (description[i]) { case '{': if (i == start) { sb.Append('{'); start = -1; } else if (start < 0) start = i + 1; break; case '}': if (start < 0) { if ((i + 1) == description.Length || description[i + 1] != '}') throw new InvalidOperationException("Invalid option description: " + description); ++i; sb.Append("}"); } else { sb.Append(description.Substring(start, i - start)); start = -1; } break; case ':': if (start < 0) goto default; start = i + 1; break; default: if (start < 0) sb.Append(description[i]); break; } } return sb.ToString(); } private static IEnumerable GetLines(string description) { if (string.IsNullOrEmpty(description)) { yield return string.Empty; yield break; } int length = 80 - OptionWidth - 1; int start = 0, end; do { end = GetLineEnd(start, length, description); char c = description[end - 1]; if (char.IsWhiteSpace(c)) --end; bool writeContinuation = end != description.Length && !IsEolChar(c); string line = description.Substring(start, end - start) + (writeContinuation ? "-" : ""); yield return line; start = end; if (char.IsWhiteSpace(c)) ++start; length = 80 - OptionWidth - 2 - 1; } while (end < description.Length); } private static bool IsEolChar(char c) { return !char.IsLetterOrDigit(c); } private static int GetLineEnd(int start, int length, string description) { int end = System.Math.Min(start + length, description.Length); int sep = -1; for (int i = start + 1; i < end; ++i) { if (description[i] == '\n') return i + 1; if (IsEolChar(description[i])) sep = i + 1; } if (sep == -1 || end == description.Length) return end; return sep; } } } ================================================ FILE: source/OctodiffAsync/OctodiffAsync.csproj ================================================  net472 7.3 Exe false false false false false false false false false AnyCPU true full false bin\ DEBUG;TRACE prompt 4 bin\OctodiffAsync.XML false AnyCPU pdbonly true bin\ TRACE prompt 4 bin\OctodiffAsync.XML false ================================================ FILE: source/OctodiffAsync/OctodiffAsyncProgram.cs ================================================ using System; using System.Collections.Generic; using System.IO; using System.Linq; using OctodiffAsync.CommandLine.Support; namespace OctodiffAsync { public class OctodiffAsyncProgram { static int Main(string[] args) { string[] commandArguments; var commandName = ExtractCommand(args, out commandArguments); var locator = new CommandLocator(); var command = locator.Find(commandName); if (command == null) { locator.Create(locator.Find("help")).Execute(commandArguments); return 4; } try { var exitCode = locator.Create(command).Execute(commandArguments); return exitCode; } catch (OptionException ex) { WriteError(ex); locator.Create(locator.Find("help")).Execute(new[] {commandName}); return 4; } catch (ArgumentException ex) { WriteError(ex); return 4; } catch (FileNotFoundException ex) { WriteError(ex); return 4; } catch (InvalidDataException ex) { WriteError(ex); return 2; } catch (IOException ex) { WriteError(ex, details: true); return 1; } catch (Exception ex) { WriteError(ex, details: true); return 3; } } static void WriteError(Exception ex, bool details = false) { Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Error: " + ex.Message); Console.ResetColor(); if (details) { Console.WriteLine(ex.ToString()); } } private static string ExtractCommand(ICollection args, out string[] remaining) { remaining = args.Count <= 1 ? new string[0] : args.Skip(1).ToArray(); return (args.FirstOrDefault() ?? string.Empty).ToLowerInvariant(); } } } ================================================ FILE: source/OctodiffAsync/Properties/AssemblyInfo.cs ================================================ using System.Reflection; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. [assembly: AssemblyTitle("OctodiffAsync")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] [assembly: AssemblyProduct("OctodiffAsync")] [assembly: AssemblyCopyright("Copyright © 2014")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] // Setting ComVisible to false makes the types in this assembly not visible // to COM components. If you need to access a type in this assembly from // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM [assembly: Guid("4c0d9c4b-8a4d-4488-9a83-553a046c18b6")] // Version information for an assembly consists of the following four values: // // Major Version // Minor Version // Build Number // Revision // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] [assembly: AssemblyVersion("1.0.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] ================================================ FILE: source/OctodiffAsync/app.config ================================================ ================================================ FILE: source/Tests.ps1 ================================================ function Assert-That ($val, $error) { if ($val -ne $true) { Write-Error "Test failed: $error" } } function Generate-RandomFile ($maxSizeInKb) { $size = Get-Random -maximum $maxSizeInKb $name = [System.Guid]::NewGuid().ToString() + ".txt" $name = (Join-Path (Resolve-Path .) $name) $data = New-Object System.IO.StreamWriter $name $rand = new-object System.Random for ($i = 0; $i -le $size; $i++) { for ($j = 0; $j -le 1020; $j++){ $c = [char]$rand.Next(65, 90) $data.Write($c) } $data.WriteLine() } $data.Close() } $here = Split-Path -Parent $MyInvocation.MyCommand.Path function Run-OctodiffSimulation($iterations) { cd $here\..\Octodiff\bin mkdir .\Temp -ErrorAction SilentlyContinue Remove-Item -Recurse -Force .\temp mkdir .\Temp -ErrorAction SilentlyContinue | Out-Null pushd .\Temp for ($i = 10; $i -le $iterations; $i++) { $name = "Package" mkdir $name -ErrorAction SilentlyContinue | Out-Null pushd $name | Out-Null for ($j = 0; $j -lt $i * 10; $j++) { Generate-RandomFile (1024) } popd $original = join-path (resolve-path .) ($name + "_orig.nupkg") dir $name | ToZip $original pushd $name | Out-Null dir | get-random -count (2) | remove-item for ($j = 0; $j -lt 3; $j++) { Generate-RandomFile (1024) } popd $name = $name + "_" + $i $newfile = join-path (resolve-path .) ($name + "_new.nupkg") dir "package" | ToZip $newfile $watch = [System.Diagnostics.Stopwatch]::StartNew() $sigfile = $original + ".octosig" & ..\Octodiff.exe signature $original $sigfile Assert-That ($LASTEXITCODE -eq 0) "Error creating signature: exit code $LASTEXITCODE" $deltafile = $original + ".octodelta" & ..\Octodiff.exe delta $sigfile $newfile $deltafile Assert-That ($LASTEXITCODE -eq 0) "Error creating delta: exit code $LASTEXITCODE" $outfile = $newfile + "_2" & ..\Octodiff.exe patch $original $deltafile $outfile Assert-That ($LASTEXITCODE -eq 0) "Error applying delta: exit code $LASTEXITCODE" Write-Host "Scenario: $i" Write-Host " Original size: $((get-item $original).Length / 1024)K" Write-Host " New size: $((get-item $newfile).Length / 1024)K" Write-Host " Signature size: $((get-item $sigfile).Length / 1024)K" Write-Host " Delta size: $((get-item $deltafile).Length / 1024)K" Write-Host " Time taken: $($watch.ElapsedMilliseconds)ms" $oldHash = (get-filehash $newfile).Hash $newHash = (Get-FileHash ($newfile + "_2")).Hash write-host " Hashes equal: $($oldHash -eq $newHash)" } popd } Set-StrictMode -V 1.0 [Reflection.Assembly]::LoadWithPartialName("WindowsBase") | Out-Null function ToZip($fileName, $relativeBaseDirectory=$null, [switch] $appendToZip=$false, $verbose=$true) { begin { $zipCreated = { (Get-Variable -ErrorAction SilentlyContinue -Name zipFile) -ne $null } $mode = [System.IO.FileMode]::Create if ($appendToZip) { $mode = [System.IO.FileMode]::Open } $zipFile = [System.IO.Packaging.Package]::Open($fileName, $mode) } process { if ((&$zipCreated) -and ([System.IO.File]::Exists($_.FullName) -eq $true)) { $zipFileName = $_.FullName if ($relativeBaseDirectory -ne $null) { #$directoryName = [System.IO.Path]::GetDirectoryName($_.FullName) $zipFileName = $_.FullName.SubString($relativeBaseDirectory.Length, $_.FullName.Length-$relativeBaseDirectory.Length) } $destFilename = [System.IO.Path]::Combine(".\\", $zipFileName) #$destFilename = $destFilename.Replace(" ", "_") $uri = New-Object Uri -ArgumentList ($destFilename, [UriKind]::Relative) $uri = [System.IO.Packaging.PackUriHelper]::CreatePartUri($uri) if ($zipFile.PartExists($uri)) { $zipFile.DeletePart($uri); } $part = $zipFile.CreatePart($uri, [string]::Empty, [System.IO.Packaging.CompressionOption]::Normal) $dest = $part.GetStream() $srcStream = New-Object System.IO.FileStream -ArgumentList ($_.FullName, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read) try { $srcStream.CopyTo($dest) } finally { $srcStream.Close() } } } end { if (&$zipCreated) { $zipFile.Close() } } } clear Run-OctodiffSimulation -iterations 10