Showing preview only (410K chars total). Download the full file or copy to clipboard to get everything.
Repository: GrzegorzBlok/FastRsyncNet
Branch: master
Commit: 961fce59da91
Files: 121
Total size: 375.9 KB
Directory structure:
gitextract_bd403tkm/
├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
└── source/
├── .editorconfig
├── FastRsync/
│ ├── Core/
│ │ ├── AggregateCopyOperationsDecorator.cs
│ │ ├── BinaryFormat.cs
│ │ ├── ChunkSignature.cs
│ │ ├── ChunkSignatureChecksumComparer.cs
│ │ ├── JsonContext.cs
│ │ ├── JsonSerializationSettings.cs
│ │ └── SupportedAlgorithms.cs
│ ├── Delta/
│ │ ├── BinaryDeltaReader.cs
│ │ ├── BinaryDeltaWriter.cs
│ │ ├── DataRange.cs
│ │ ├── DeltaApplier.cs
│ │ ├── DeltaBuilder.cs
│ │ ├── DeltaMetadata.cs
│ │ ├── IDeltaReader.cs
│ │ └── IDeltaWriter.cs
│ ├── Diagnostics/
│ │ ├── ConsoleProgressReporter.cs
│ │ └── ProgressReport.cs
│ ├── FastRsync.csproj
│ ├── Hash/
│ │ ├── Adler32RollingChecksum.cs
│ │ ├── Adler32RollingChecksumV2.cs
│ │ ├── Adler32RollingChecksumV3.cs
│ │ ├── CryptographyHashAlgorithmWrapper.cs
│ │ ├── IHashAlgorithm.cs
│ │ ├── IRollingChecksum.cs
│ │ └── NonCryptographicHashAlgorithmWrapper.cs
│ └── Signature/
│ ├── ISignatureReader.cs
│ ├── ISignatureWriter.cs
│ ├── Signature.cs
│ ├── SignatureBuilder.cs
│ ├── SignatureReader.cs
│ └── SignatureWriter.cs
├── FastRsync.BackwardCompatibilityTests/
│ ├── BackwardCompatibilityTests.cs
│ └── FastRsync.BackwardCompatibilityTests.csproj
├── FastRsync.Benchmarks/
│ ├── BuildPatchBenchmark.cs
│ ├── FastRsync.Benchmarks.csproj
│ ├── HashBenchmark.cs
│ ├── Program.cs
│ ├── RollingChecksumBenchmark.cs
│ └── SignatureBenchmark.cs
├── FastRsync.Compression/
│ ├── FastRsync.Compression.csproj
│ └── GZip.cs
├── FastRsync.Tests/
│ ├── Adler32RollingChecksumAlgorithmsTests.cs
│ ├── Adler32RollingChecksumTests.cs
│ ├── Adler32RollingChecksumV2Tests.cs
│ ├── Adler32RollingChecksumV3Tests.cs
│ ├── BackwardCompatibilityTests.cs
│ ├── CommonAsserts.cs
│ ├── DeltaReaderTests.cs
│ ├── FastRsync.Tests.csproj
│ ├── FastRsyncLegacy/
│ │ ├── BinaryDeltaReaderLegacy.cs
│ │ ├── DeltaMetadataLegacy.cs
│ │ ├── IDeltaReaderLegacy.cs
│ │ └── NewtonsoftJsonSerializationSettings.cs
│ ├── GZipTests.cs
│ ├── HashTests.cs
│ ├── OctodiffLegacy/
│ │ ├── IOctodiffDeltaWriter.cs
│ │ ├── IOctodiffSignatureWriter.cs
│ │ ├── OctodiffAggregateCopyOperationsDecorator.cs
│ │ ├── OctodiffBinaryDeltaWriter.cs
│ │ ├── OctodiffBinaryFormat.cs
│ │ ├── OctodiffDeltaBuilder.cs
│ │ ├── OctodiffSignature.cs
│ │ ├── OctodiffSignatureBuilder.cs
│ │ └── OctodiffSignatureReader.cs
│ ├── PatchingAsyncTests.cs
│ ├── PatchingBigFilesTests.cs
│ ├── PatchingSyncTests.cs
│ ├── SignatureBuilderAsyncRandomDataTests.cs
│ ├── SignatureBuilderSyncRandomDataTests.cs
│ ├── SignatureBuilderTests.cs
│ ├── SignatureReaderTests.cs
│ └── Utils.cs
├── FastRsync.sln
├── Octodiff/
│ ├── CommandLine/
│ │ ├── DeltaCommand.cs
│ │ ├── ExplainDeltaCommand.cs
│ │ ├── HelpCommand.cs
│ │ ├── PatchCommand.cs
│ │ ├── SignatureCommand.cs
│ │ └── Support/
│ │ ├── CommandAttribute.cs
│ │ ├── CommandException.cs
│ │ ├── CommandLocator.cs
│ │ ├── ICommand.cs
│ │ ├── ICommandLocator.cs
│ │ ├── ICommandMetadata.cs
│ │ └── NDesk.Options.cs
│ ├── Octodiff.csproj
│ ├── OctodiffProgram.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ └── app.config
├── Octodiff.Tests/
│ ├── DeltaFixture.cs
│ ├── HelpFixture.cs
│ ├── Octodiff.Tests.csproj
│ ├── PackageGenerator.cs
│ ├── PatchFixture.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ ├── SignatureFixture.cs
│ ├── Timings.cs
│ └── Util/
│ ├── CommandLineFixture.cs
│ └── SilentProcessRunner.cs
├── OctodiffAsync/
│ ├── CommandLine/
│ │ ├── DeltaCommand.cs
│ │ ├── ExplainDeltaCommand.cs
│ │ ├── HelpCommand.cs
│ │ ├── PatchCommand.cs
│ │ ├── SignatureCommand.cs
│ │ └── Support/
│ │ ├── CommandAttribute.cs
│ │ ├── CommandException.cs
│ │ ├── CommandLocator.cs
│ │ ├── ICommand.cs
│ │ ├── ICommandLocator.cs
│ │ ├── ICommandMetadata.cs
│ │ └── NDesk.Options.cs
│ ├── OctodiffAsync.csproj
│ ├── OctodiffAsyncProgram.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ └── app.config
└── Tests.ps1
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain
================================================
FILE: .gitignore
================================================
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.sln.docstates
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
x64/
build/
bld/
[Bb]in/
[Oo]bj/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
#NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opensdf
*.sdf
*.cachefile
# Visual Studio profiler
*.psess
*.vsp
*.vspx
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding addin-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
*.ncrunch*
_NCrunch_*
.*crunch*.local.xml
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# NuGet Packages Directory
packages/
## TODO: If the tool you use requires repositories.config uncomment the next line
#!packages/repositories.config
# Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets
# This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented)
!packages/build/
# Windows Azure Build Output
csx/
*.build.csdef
# Windows Store app package directory
AppPackages/
# Others
sql/
*.Cache
ClientBin/
[Ss]tyle[Cc]op.*
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file to a newer
# Visual Studio version. Backup files are not needed, because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
/.vs
/source/.vs/FastRsync/v15/Server/sqlite3
/source/.vs
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# FastRsyncNet - C# delta syncing library
The Fast Rsync .NET library is Rsync implementation derived from [Octodiff](https://github.com/OctopusDeploy/Octodiff) tool.
Unlike the Octodiff which is based on SHA1 algorithm, the FastRsyncNet allows a variety of hashing algorithms to choose from.
The default one, that is xxHash64, offers significantly faster calculations and smaller signature size than the SHA1, while still providing sufficient quality of hash results.
FastRsyncNet supports also SHA1 and is 100% compatible with signatures and deltas produced by Octodiff.
Since version 2.0.0 the signature and delta format has changed. FastRsyncNet 2.x is still able to work with signatures and deltas from FastRsync 1.x and Octodiff. However, files made with FastRsyncNet 2.x are not going to be recognized by FastRsyncNet 1.x.
## Install [](https://www.nuget.org/packages/FastRsyncNet/)
Add To project via NuGet:
1. Right click on a project and click 'Manage NuGet Packages'.
2. Search for 'FastRsyncNet' and click 'Install'.
## Examples
### Calculating signature
```csharp
using FastRsync.Signature;
...
var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32RollingV3());
using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read))
{
signatureBuilder.Build(basisStream, new SignatureWriter(signatureStream));
}
```
### Calculating delta
```csharp
using FastRsync.Delta;
...
var delta = new DeltaBuilder();
builder.ProgressReport = new ConsoleProgressReporter();
using (var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var signatureStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read))
{
delta.BuildDelta(newFileStream, new SignatureReader(signatureStream, delta.ProgressReporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
}
```
### Patching (applying delta)
```csharp
using FastRsync.Delta;
...
var delta = new DeltaApplier
{
SkipHashCheck = true
};
using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var newFileStream = new FileStream(newFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read))
{
delta.Apply(basisStream, new BinaryDeltaReader(deltaStream, progressReporter), newFileStream);
}
```
### Calculating signature on Azure blobs
FastRsyncNet might not work on Azure Storage emulator due to issues with stream seeking.
```csharp
using FastRsync.Signature;
...
var storageAccount = CloudStorageAccount.Parse("azure storage connectionstring");
var blobClient = storageAccount.CreateCloudBlobClient();
var blobsContainer = blobClient.GetContainerReference("containerName");
var basisBlob = blobsContainer.GetBlockBlobReference("blobName");
var signatureBlob = container.GetBlockBlobReference("blob_signature");
var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32RollingV3());
using (var signatureStream = await signatureBlob.OpenWriteAsync())
using (var basisStream = await basisBlob.OpenReadAsync())
{
await signatureBuilder.BuildAsync(basisStream, new SignatureWriter(signatureStream));
}
```
## Available algorithms and relative performance
Following signature hashing algorithms are available:
* XxHash64 - default algorithm, signature size 6.96 MB, signature calculation time 5209 ms.
* XxHash3 - signature size 6.96 MB, signature calculation time 5024 ms. For all new use cases, use this one.
* SHA1 - signature size 12.9 MB, signature calculation time 6519 ms.
* MD5 - originally used in Rsync program, signature size 10.9 MB, signature calculation time 6767 ms.
The signature sizes and calculation times are to provide some insights on relative perfomance. The real perfomance on your system will vary greatly. The benchmark had been run against 0.99 GB file.
Following rolling checksum algorithms are available:
* Adler32RollingChecksum - default algorithm, it uses low level optimization that makes it faster but provides worse quality of checksum.
* Adler32RollingChecksumV2 - Obsolete. It has a bug that - while does not make any data incorrect - results in unnecessary big deltas. Do not use it, unless you need to due to the backward compatibility. It is the original (but incorrectly implemented) Adler32 algorithm implementation (slower but better quality of checksum).
* Adler32RollingChecksumV3 - for all new use cases, use this one. It is fast and has best quality of checksum.
## GZip compression that is rsync compatible [](https://www.nuget.org/packages/FastRsyncNet.Compression/)
If you synchronize a compressed file, a small change in a compressed file may force rsync algorithm to synchronize whole compressed file, instead of just the changed blocks. To fix this, a custom GZip compression method may be used that periodically reset the compressor state to make it block-sync friendly. Install [FastRsyncNet.Compression](https://www.nuget.org/packages/FastRsyncNet.Compression/) package and use following method:
```csharp
FastRsync.Compression.GZip.Compress(Stream sourceStream, Stream destStream)
```
To uncompress you may use any GZip method (e.g. System.IO.Compression.GZipStream).
================================================
FILE: source/.editorconfig
================================================
; EditorConfig to support per-solution formatting.
; Use the EditorConfig VS add-in to make this work.
; http://editorconfig.org/
; This is the default for the codeline.
root = true
[*]
end_of_line = CRLF
[*.{cs,props,targets,xml,nuspec}]
indent_style = space
indent_size = 4
[*.{config}]
indent_style = space
indent_size = 2
[*.{csproj,resx}]
indent_style = space
indent_size = 2
================================================
FILE: source/FastRsync/Core/AggregateCopyOperationsDecorator.cs
================================================
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Delta;
namespace FastRsync.Core
{
// This decorator turns any sequential copy operations into a single operation, reducing
// the size of the delta file.
// For example:
// Copy: 0x0000 - 0x0400
// Copy: 0x0401 - 0x0800
// Copy: 0x0801 - 0x0C00
// Gets turned into:
// Copy: 0x0000 - 0x0C00
public class AggregateCopyOperationsDecorator : IDeltaWriter
{
private readonly IDeltaWriter decorated;
private DataRange bufferedCopy;
public AggregateCopyOperationsDecorator(IDeltaWriter decorated)
{
this.decorated = decorated;
}
public void WriteDataCommand(Stream source, long offset, long length)
{
FlushCurrentCopyCommand();
decorated.WriteDataCommand(source, offset, length);
}
public async Task WriteDataCommandAsync(Stream source, long offset, long length, CancellationToken cancellationToken)
{
FlushCurrentCopyCommand();
await decorated.WriteDataCommandAsync(source, offset, length, cancellationToken).ConfigureAwait(false);
}
public void WriteMetadata(DeltaMetadata metadata)
{
decorated.WriteMetadata(metadata);
}
public void WriteCopyCommand(DataRange chunk)
{
if (bufferedCopy.Length > 0 && bufferedCopy.StartOffset + bufferedCopy.Length == chunk.StartOffset)
{
bufferedCopy.Length += chunk.Length;
}
else
{
FlushCurrentCopyCommand();
bufferedCopy = chunk;
}
}
private void FlushCurrentCopyCommand()
{
if (bufferedCopy.Length <= 0)
{
return;
}
decorated.WriteCopyCommand(bufferedCopy);
bufferedCopy = new DataRange();
}
public void Finish()
{
FlushCurrentCopyCommand();
decorated.Finish();
}
}
}
================================================
FILE: source/FastRsync/Core/BinaryFormat.cs
================================================
using System.Text;
namespace FastRsync.Core
{
internal class BinaryFormat
{
public const int SignatureFormatHeaderLength = 7; // OCTOSIG or FRSNCSG
public const int DeltaFormatHeaderLength = 9; // OCTODELTA or FRSNCDLTA
public const byte CopyCommand = 0x60;
public const byte DataCommand = 0x80;
}
internal class OctoBinaryFormat
{
public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG");
public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA");
public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>");
public const byte Version = 0x01;
}
internal class FastRsyncBinaryFormat
{
public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("FRSNCSG");
public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("FRSNCDLTA");
public const byte Version = 0x01;
}
}
================================================
FILE: source/FastRsync/Core/ChunkSignature.cs
================================================
using System;
namespace FastRsync.Core
{
public class ChunkSignature
{
public long StartOffset; // 8 (but not included in the file on disk)
public short Length; // 2
public byte[] Hash; // depending on hash (20 for SHA1, 8 for xxHash64)
public UInt32 RollingChecksum; // 4
public override string ToString()
{
return string.Format("{0,6}:{1,6} |{2,20}| {3}", StartOffset, Length, RollingChecksum, BitConverter.ToString(Hash).ToLowerInvariant().Replace("-", ""));
}
}
}
================================================
FILE: source/FastRsync/Core/ChunkSignatureChecksumComparer.cs
================================================
using System.Collections.Generic;
namespace FastRsync.Core
{
class ChunkSignatureChecksumComparer : IComparer<ChunkSignature>
{
public int Compare(ChunkSignature x, ChunkSignature y)
{
var comparison = x.RollingChecksum.CompareTo(y.RollingChecksum);
return comparison == 0 ? x.StartOffset.CompareTo(y.StartOffset) : comparison;
}
}
}
================================================
FILE: source/FastRsync/Core/JsonContext.cs
================================================
using FastRsync.Delta;
using FastRsync.Signature;
namespace FastRsync.Core
{
#if NET7_0_OR_GREATER
using System.Text.Json.Serialization;
/// <summary>
/// Provides AOT-safe JSON serialization metadata for FastRsync types.
/// </summary>
[JsonSerializable(typeof(DeltaMetadata))]
[JsonSerializable(typeof(SignatureMetadata))]
[JsonSourceGenerationOptions(
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
)]
public partial class JsonContextCore : JsonSerializerContext;
#endif
}
================================================
FILE: source/FastRsync/Core/JsonSerializationSettings.cs
================================================
using System.Text.Json;
using System.Text.Json.Serialization;
namespace FastRsync.Core
{
public class JsonSerializationSettings
{
static JsonSerializationSettings()
{
JsonSettings = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
}
public static JsonSerializerOptions JsonSettings { get; }
}
}
================================================
FILE: source/FastRsync/Core/SupportedAlgorithms.cs
================================================
using System;
using System.IO.Hashing;
using System.Security.Cryptography;
using FastRsync.Hash;
namespace FastRsync.Core
{
public static class SupportedAlgorithms
{
public static class Hashing
{
public static IHashAlgorithm Sha1()
{
return new CryptographyHashAlgorithmWrapper("SHA1", SHA1.Create());
}
public static IHashAlgorithm Md5()
{
return new CryptographyHashAlgorithmWrapper("MD5", MD5.Create());
}
public static IHashAlgorithm XxHash()
{
return new NonCryptographicHashAlgorithmWrapper("XXH64", new XxHash64());
}
public static IHashAlgorithm XxHash3()
{
return new NonCryptographicHashAlgorithmWrapper("XXH3", new XxHash3());
}
public static IHashAlgorithm Default()
{
return XxHash();
}
public static IHashAlgorithm Create(string algorithmName)
{
switch (algorithmName)
{
case "XXH64":
return XxHash();
case "MD5":
return Md5();
case "XXH3":
return XxHash3();
case "SHA1":
return Sha1();
}
throw new NotSupportedException($"The hash algorithm '{algorithmName}' is not supported");
}
}
public static class Checksum
{
public static IRollingChecksum Adler32Rolling() { return new Adler32RollingChecksum(); }
[Obsolete("Adler32V2 has buggy mod operation implemented. See https://github.com/GrzegorzBlok/FastRsyncNet/issues/20")]
public static IRollingChecksum Adler32RollingV2() { return new Adler32RollingChecksumV2(); }
public static IRollingChecksum Adler32RollingV3() { return new Adler32RollingChecksumV3(); }
public static IRollingChecksum Default()
{
return Adler32Rolling();
}
public static IRollingChecksum Create(string algorithm)
{
switch (algorithm)
{
case "Adler32":
return Adler32Rolling();
case "Adler32V2":
return Adler32RollingV2();
case "Adler32V3":
return Adler32RollingV3();
default:
throw new NotSupportedException($"The rolling checksum algorithm '{algorithm}' is not supported");
}
}
}
}
}
================================================
FILE: source/FastRsync/Delta/BinaryDeltaReader.cs
================================================
using System;
using System.Collections;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Diagnostics;
using FastRsync.Hash;
using FastRsync.Signature;
namespace FastRsync.Delta
{
public class BinaryDeltaReader : IDeltaReader
{
private readonly BinaryReader reader;
private readonly IProgress<ProgressReport> progressReport;
private byte[] expectedHash;
private IHashAlgorithm hashAlgorithm;
private readonly int readBufferSize;
public BinaryDeltaReader(Stream stream, IProgress<ProgressReport> progressHandler,
int readBufferSize = 4 * 1024 * 1024)
{
this.reader = new BinaryReader(stream);
this.progressReport = progressHandler;
this.readBufferSize = readBufferSize;
}
private DeltaMetadata _metadata;
private RsyncFormatType type;
public DeltaMetadata Metadata
{
get
{
ReadMetadata();
return _metadata;
}
}
public RsyncFormatType Type
{
get
{
ReadMetadata();
return type;
}
}
public byte[] ExpectedHash
{
get
{
ReadMetadata();
return expectedHash;
}
}
public IHashAlgorithm HashAlgorithm
{
get
{
ReadMetadata();
return hashAlgorithm;
}
}
private void ReadMetadata()
{
if (_metadata != null)
return;
reader.BaseStream.Seek(0, SeekOrigin.Begin);
var header = reader.ReadBytes(BinaryFormat.DeltaFormatHeaderLength);
if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.DeltaHeader, header))
{
ReadFastRsyncDeltaHeader();
return;
}
if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.DeltaHeader, header))
{
ReadOctoDeltaHeader();
return;
}
throw new InvalidDataException("The delta file uses a different file format than this program can handle.");
}
private void ReadFastRsyncDeltaHeader()
{
var version = reader.ReadByte();
if (version != FastRsyncBinaryFormat.Version)
throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
var metadataStr = reader.ReadString();
#if NET7_0_OR_GREATER
_metadata = JsonSerializer.Deserialize(metadataStr, JsonContextCore.Default.DeltaMetadata);
#else
_metadata = JsonSerializer.Deserialize<DeltaMetadata>(metadataStr, JsonSerializationSettings.JsonSettings);
#endif
hashAlgorithm = SupportedAlgorithms.Hashing.Create(_metadata.HashAlgorithm);
expectedHash = Convert.FromBase64String(_metadata.ExpectedFileHash);
type = RsyncFormatType.FastRsync;
}
private void ReadOctoDeltaHeader()
{
var version = reader.ReadByte();
if (version != OctoBinaryFormat.Version)
throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
var hashAlgorithmName = reader.ReadString();
hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName);
var hashLength = reader.ReadInt32();
expectedHash = reader.ReadBytes(hashLength);
var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length);
if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta))
throw new InvalidDataException("The delta file appears to be corrupt.");
_metadata = new DeltaMetadata
{
HashAlgorithm = hashAlgorithmName,
ExpectedFileHashAlgorithm = hashAlgorithmName,
ExpectedFileHash = Convert.ToBase64String(expectedHash)
};
type = RsyncFormatType.Octodiff;
}
public void Apply(
Action<byte[]> writeData,
Action<long, long> copy)
{
var fileLength = reader.BaseStream.Length;
ReadMetadata();
while (reader.BaseStream.Position != fileLength)
{
var b = reader.ReadByte();
progressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.ApplyingDelta,
CurrentPosition = reader.BaseStream.Position,
Total = fileLength
});
if (b == BinaryFormat.CopyCommand)
{
var start = reader.ReadInt64();
var length = reader.ReadInt64();
copy(start, length);
}
else if (b == BinaryFormat.DataCommand)
{
var length = reader.ReadInt64();
long soFar = 0;
while (soFar < length)
{
var bytes = reader.ReadBytes((int)Math.Min(length - soFar, readBufferSize));
soFar += bytes.Length;
writeData(bytes);
}
}
}
}
public Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> copy) =>
ApplyAsync(writeData, copy, CancellationToken.None);
public async Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> copy,
CancellationToken cancellationToken)
{
var fileLength = reader.BaseStream.Length;
ReadMetadata();
var buffer = new byte[readBufferSize];
while (reader.BaseStream.Position != fileLength)
{
var b = reader.ReadByte();
progressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.ApplyingDelta,
CurrentPosition = reader.BaseStream.Position,
Total = fileLength
});
if (b == BinaryFormat.CopyCommand)
{
var start = reader.ReadInt64();
var length = reader.ReadInt64();
await copy(start, length).ConfigureAwait(false);
}
else if (b == BinaryFormat.DataCommand)
{
var length = reader.ReadInt64();
long soFar = 0;
while (soFar < length)
{
var bytesRead = await reader.BaseStream
.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken)
.ConfigureAwait(false);
var bytes = buffer;
if (bytesRead != buffer.Length)
{
bytes = new byte[bytesRead];
Array.Copy(buffer, bytes, bytesRead);
}
soFar += bytes.Length;
await writeData(bytes).ConfigureAwait(false);
}
}
}
}
}
}
================================================
FILE: source/FastRsync/Delta/BinaryDeltaWriter.cs
================================================
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
namespace FastRsync.Delta
{
public class BinaryDeltaWriter : IDeltaWriter
{
private readonly Stream deltaStream;
private readonly BinaryWriter writer;
private readonly int readWriteBufferSize;
public BinaryDeltaWriter(Stream stream, int readWriteBufferSize = 1024 * 1024)
{
deltaStream = stream;
writer = new BinaryWriter(stream);
this.readWriteBufferSize = readWriteBufferSize;
}
public void WriteMetadata(DeltaMetadata metadata)
{
writer.Write(FastRsyncBinaryFormat.DeltaHeader);
writer.Write(FastRsyncBinaryFormat.Version);
#if NET7_0_OR_GREATER
var metadataStr = JsonSerializer.Serialize(metadata, JsonContextCore.Default.DeltaMetadata);
#else
var metadataStr = JsonSerializer.Serialize(metadata, JsonSerializationSettings.JsonSettings);
#endif
writer.Write(metadataStr);
}
public void WriteCopyCommand(DataRange segment)
{
writer.Write(BinaryFormat.CopyCommand);
writer.Write(segment.StartOffset);
writer.Write(segment.Length);
}
public void WriteDataCommand(Stream source, long offset, long length)
{
writer.Write(BinaryFormat.DataCommand);
writer.Write(length);
var originalPosition = source.Position;
try
{
source.Seek(offset, SeekOrigin.Begin);
var buffer = new byte[(int)Math.Min(length, readWriteBufferSize)];
int read;
long soFar = 0;
while ((read = source.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0)
{
soFar += read;
writer.Write(buffer, 0, read);
}
}
finally
{
source.Seek(originalPosition, SeekOrigin.Begin);
}
}
public async Task WriteDataCommandAsync(Stream source, long offset, long length,
CancellationToken cancellationToken)
{
writer.Write(BinaryFormat.DataCommand);
writer.Write(length);
var originalPosition = source.Position;
try
{
source.Seek(offset, SeekOrigin.Begin);
var buffer = new byte[(int)Math.Min(length, readWriteBufferSize)];
int read;
long soFar = 0;
while ((read = await source
.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken)
.ConfigureAwait(false)) > 0)
{
soFar += read;
await deltaStream.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false);
}
}
finally
{
source.Seek(originalPosition, SeekOrigin.Begin);
}
}
public void Finish()
{
}
}
}
================================================
FILE: source/FastRsync/Delta/DataRange.cs
================================================
namespace FastRsync.Delta
{
public struct DataRange
{
public DataRange(long startOffset, long length)
{
StartOffset = startOffset;
Length = length;
}
public long StartOffset;
public long Length;
}
}
================================================
FILE: source/FastRsync/Delta/DeltaApplier.cs
================================================
using System;
using System.Collections;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
namespace FastRsync.Delta
{
public class DeltaApplier
{
private readonly int readBufferSize;
public DeltaApplier(int readBufferSize = 4 * 1024 * 1024)
{
SkipHashCheck = false;
this.readBufferSize = readBufferSize;
}
public bool SkipHashCheck { get; set; }
public void Apply(Stream basisFileStream, IDeltaReader delta, Stream outputStream)
{
var buffer = new byte[readBufferSize];
delta.Apply(
writeData: (data) => outputStream.Write(data, 0, data.Length),
copy: (startPosition, length) =>
{
basisFileStream.Seek(startPosition, SeekOrigin.Begin);
int read;
long soFar = 0;
while ((read = basisFileStream.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0)
{
soFar += read;
outputStream.Write(buffer, 0, read);
}
});
if (!SkipHashCheck)
{
if (!HashCheck(delta, outputStream))
{
throw new InvalidDataException(
$"Verification of the patched file failed. The {delta.HashAlgorithm.Name} hash of the patch result file, and the file that was used as input for the delta, do not match. This can happen if the basis file changed since the signatures were calculated.");
}
}
}
public Task ApplyAsync(Stream basisFileStream, IDeltaReader delta, Stream outputStream) =>
ApplyAsync(basisFileStream, delta, outputStream, CancellationToken.None);
public async Task ApplyAsync(Stream basisFileStream, IDeltaReader delta, Stream outputStream, CancellationToken cancellationToken)
{
var buffer = new byte[readBufferSize];
await delta.ApplyAsync(
writeData: async (data) => await outputStream.WriteAsync(data, 0, data.Length, cancellationToken).ConfigureAwait(false),
copy: async (startPosition, length) =>
{
basisFileStream.Seek(startPosition, SeekOrigin.Begin);
int read;
long soFar = 0;
while ((read = await basisFileStream.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
soFar += read;
await outputStream.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false);
}
}, cancellationToken).ConfigureAwait(false);
if (!SkipHashCheck)
{
if (!await HashCheckAsync(delta, outputStream, cancellationToken).ConfigureAwait(false))
{
throw new InvalidDataException(
$"Verification of the patched file failed. The {delta.Metadata.ExpectedFileHashAlgorithm} hash of the patch result file, and the file that was used as input for the delta, do not match. This can happen if the basis file changed since the signatures were calculated.");
}
}
}
public bool HashCheck(IDeltaReader delta, Stream outputStream)
{
outputStream.Seek(0, SeekOrigin.Begin);
var sourceFileHash = delta.ExpectedHash;
var algorithm = SupportedAlgorithms.Hashing.Create(delta.Metadata.ExpectedFileHashAlgorithm);
var actualHash = algorithm.ComputeHash(outputStream);
return StructuralComparisons.StructuralEqualityComparer.Equals(sourceFileHash, actualHash);
}
public Task<bool> HashCheckAsync(IDeltaReader delta, Stream outputStream) => HashCheckAsync(delta, outputStream, CancellationToken.None);
public async Task<bool> HashCheckAsync(IDeltaReader delta, Stream outputStream, CancellationToken cancellationToken)
{
outputStream.Seek(0, SeekOrigin.Begin);
var sourceFileHash = delta.ExpectedHash;
var algorithm = SupportedAlgorithms.Hashing.Create(delta.Metadata.ExpectedFileHashAlgorithm);
var actualHash = await algorithm.ComputeHashAsync(outputStream, cancellationToken).ConfigureAwait(false);
return StructuralComparisons.StructuralEqualityComparer.Equals(sourceFileHash, actualHash);
}
}
}
================================================
FILE: source/FastRsync/Delta/DeltaBuilder.cs
================================================
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Diagnostics;
using FastRsync.Signature;
namespace FastRsync.Delta
{
public class DeltaBuilder
{
private readonly int readBufferSize;
public DeltaBuilder(int readBufferSize = 4 * 1024 * 1024)
{
ProgressReport = null;
this.readBufferSize = readBufferSize;
}
public IProgress<ProgressReport> ProgressReport { get; set; }
public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter)
{
var newFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5();
newFileStream.Seek(0, SeekOrigin.Begin);
var newFileHash = newFileVerificationHashAlgorithm.ComputeHash(newFileStream);
newFileStream.Seek(0, SeekOrigin.Begin);
var signature = signatureReader.ReadSignature();
var chunks = OrderChunksByChecksum(signature.Chunks);
var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize);
deltaWriter.WriteMetadata(new DeltaMetadata
{
HashAlgorithm = signature.HashAlgorithm.Name,
ExpectedFileHashAlgorithm = newFileVerificationHashAlgorithm.Name,
ExpectedFileHash = Convert.ToBase64String(newFileHash),
BaseFileHash = signature.Metadata.BaseFileHash,
BaseFileHashAlgorithm = signature.Metadata.BaseFileHashAlgorithm
});
var checksumAlgorithm = signature.RollingChecksumAlgorithm;
var buffer = new byte[readBufferSize];
long lastMatchPosition = 0;
var fileSize = newFileStream.Length;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingDelta,
CurrentPosition = 0,
Total = fileSize
});
while (true)
{
var startPosition = newFileStream.Position;
var read = newFileStream.Read(buffer, 0, buffer.Length);
if (read <= 0)
break;
uint checksum = 0;
var remainingPossibleChunkSize = maxChunkSize;
for (var i = 0; i < read - minChunkSize + 1; i++)
{
var readSoFar = startPosition + i;
var remainingBytes = read - i;
if (remainingBytes < maxChunkSize)
{
remainingPossibleChunkSize = minChunkSize;
}
if (i == 0 || remainingBytes < maxChunkSize)
{
checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
}
else
{
var remove = buffer[i - 1];
var add = buffer[i + remainingPossibleChunkSize - 1];
checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
}
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingDelta,
CurrentPosition = readSoFar,
Total = fileSize
});
if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
continue;
if (!chunkMap.TryGetValue(checksum, out var startIndex))
continue;
for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
{
var chunk = chunks[j];
var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);
if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash))
{
readSoFar += remainingPossibleChunkSize;
var missing = readSoFar - lastMatchPosition;
if (missing > remainingPossibleChunkSize)
{
deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize);
}
deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
lastMatchPosition = readSoFar;
break;
}
}
}
if (read < buffer.Length)
{
break;
}
newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
}
if (newFileStream.Length != lastMatchPosition)
{
deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition);
}
deltaWriter.Finish();
}
public Task BuildDeltaAsync(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter) =>
BuildDeltaAsync(newFileStream, signatureReader, deltaWriter, CancellationToken.None);
public async Task BuildDeltaAsync(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter, CancellationToken cancellationToken)
{
var newFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5();
newFileStream.Seek(0, SeekOrigin.Begin);
var newFileHash = await newFileVerificationHashAlgorithm.ComputeHashAsync(newFileStream, cancellationToken).ConfigureAwait(false);
newFileStream.Seek(0, SeekOrigin.Begin);
var signature = signatureReader.ReadSignature();
var chunks = OrderChunksByChecksum(signature.Chunks);
var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize);
deltaWriter.WriteMetadata(new DeltaMetadata
{
HashAlgorithm = signature.HashAlgorithm.Name,
ExpectedFileHashAlgorithm = newFileVerificationHashAlgorithm.Name,
ExpectedFileHash = Convert.ToBase64String(newFileHash),
BaseFileHash = signature.Metadata.BaseFileHash,
BaseFileHashAlgorithm = signature.Metadata.BaseFileHashAlgorithm
});
var checksumAlgorithm = signature.RollingChecksumAlgorithm;
var buffer = new byte[readBufferSize];
long lastMatchPosition = 0;
var fileSize = newFileStream.Length;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingDelta,
CurrentPosition = 0,
Total = fileSize
});
while (true)
{
var startPosition = newFileStream.Position;
var read = await newFileStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false);
if (read <= 0)
break;
uint checksum = 0;
var remainingPossibleChunkSize = maxChunkSize;
for (var i = 0; i < read - minChunkSize + 1; i++)
{
var readSoFar = startPosition + i;
var remainingBytes = read - i;
if (remainingBytes < maxChunkSize)
{
remainingPossibleChunkSize = minChunkSize;
}
if (i == 0 || remainingBytes < maxChunkSize)
{
checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
}
else
{
var remove = buffer[i - 1];
var add = buffer[i + remainingPossibleChunkSize - 1];
checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
}
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingDelta,
CurrentPosition = readSoFar,
Total = fileSize
});
if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
continue;
if (!chunkMap.TryGetValue(checksum, out var startIndex))
continue;
for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
{
var chunk = chunks[j];
var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);
if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash))
{
readSoFar += remainingPossibleChunkSize;
var missing = readSoFar - lastMatchPosition;
if (missing > remainingPossibleChunkSize)
{
await deltaWriter.WriteDataCommandAsync(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize, cancellationToken).ConfigureAwait(false);
}
deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
lastMatchPosition = readSoFar;
break;
}
}
}
if (read < buffer.Length)
{
break;
}
newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
}
if (newFileStream.Length != lastMatchPosition)
{
await deltaWriter.WriteDataCommandAsync(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition, cancellationToken).ConfigureAwait(false);
}
deltaWriter.Finish();
}
private static List<ChunkSignature> OrderChunksByChecksum(List<ChunkSignature> chunks)
{
chunks.Sort(new ChunkSignatureChecksumComparer());
return chunks;
}
private Dictionary<uint, int> CreateChunkMap(IList<ChunkSignature> chunks, out int maxChunkSize, out int minChunkSize)
{
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.CreatingChunkMap,
CurrentPosition = 0,
Total = chunks.Count
});
maxChunkSize = 0;
minChunkSize = int.MaxValue;
var chunkMap = new Dictionary<uint, int>();
for (var i = 0; i < chunks.Count; i++)
{
var chunk = chunks[i];
if (chunk.Length > maxChunkSize)
{
maxChunkSize = chunk.Length;
}
if (chunk.Length < minChunkSize)
{
minChunkSize = chunk.Length;
}
if (!chunkMap.ContainsKey(chunk.RollingChecksum))
{
chunkMap[chunk.RollingChecksum] = i;
}
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.CreatingChunkMap,
CurrentPosition = i,
Total = chunks.Count
});
}
return chunkMap;
}
}
}
================================================
FILE: source/FastRsync/Delta/DeltaMetadata.cs
================================================
namespace FastRsync.Delta
{
public class DeltaMetadata
{
public string HashAlgorithm { get; set; }
public string ExpectedFileHashAlgorithm { get; set; }
public string ExpectedFileHash { get; set; }
public string BaseFileHashAlgorithm { get; set; }
public string BaseFileHash { get; set; }
}
}
================================================
FILE: source/FastRsync/Delta/IDeltaReader.cs
================================================
using System;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Hash;
using FastRsync.Signature;
namespace FastRsync.Delta
{
public interface IDeltaReader
{
byte[] ExpectedHash { get; }
IHashAlgorithm HashAlgorithm { get; }
DeltaMetadata Metadata { get; }
RsyncFormatType Type { get; }
void Apply(Action<byte[]> writeData, Action<long, long> copy);
Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> copy);
Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> copy, CancellationToken cancellationToken);
}
}
================================================
FILE: source/FastRsync/Delta/IDeltaWriter.cs
================================================
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Hash;
namespace FastRsync.Delta
{
public interface IDeltaWriter
{
void WriteMetadata(DeltaMetadata metadata);
void WriteCopyCommand(DataRange segment);
void WriteDataCommand(Stream source, long offset, long length);
Task WriteDataCommandAsync(Stream source, long offset, long length, CancellationToken cancellationToken);
void Finish();
}
}
================================================
FILE: source/FastRsync/Diagnostics/ConsoleProgressReporter.cs
================================================
using System;
namespace FastRsync.Diagnostics
{
public class ConsoleProgressReporter : IProgress<ProgressReport>
{
private ProgressOperationType currentOperation;
private int progressPercentage;
public void Report(ProgressReport progress)
{
var percent = (int)((double)progress.CurrentPosition / progress.Total * 100d + 0.5);
if (currentOperation != progress.Operation)
{
progressPercentage = -1;
currentOperation = progress.Operation;
}
if (progressPercentage != percent && percent % 10 == 0)
{
progressPercentage = percent;
Console.WriteLine("{0}: {1}%", currentOperation, percent);
}
}
}
}
================================================
FILE: source/FastRsync/Diagnostics/ProgressReport.cs
================================================
namespace FastRsync.Diagnostics
{
public enum ProgressOperationType
{
BuildingSignatures,
HashingFile,
ReadingSignature,
CreatingChunkMap,
BuildingDelta,
ApplyingDelta
}
public sealed class ProgressReport
{
public ProgressOperationType Operation { get; internal set; }
public long CurrentPosition { get; internal set; }
public long Total { get; internal set; }
}
}
================================================
FILE: source/FastRsync/FastRsync.csproj
================================================
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup Condition="'$(TargetFramework)' == 'net7.0'">
<LangVersion>12.0</LangVersion>
</PropertyGroup>
<PropertyGroup>
<TargetFrameworks>net462;netstandard2.0;net7.0;net8.0;net9.0;net10.0</TargetFrameworks>
<PackageId>FastRsyncNet</PackageId>
<Authors>Grzegorz Blok</Authors>
<PackageProjectUrl>https://github.com/GrzegorzBlok/FastRsyncNet</PackageProjectUrl>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<AssemblyVersion>2.4.5</AssemblyVersion>
<FileVersion>2.4.5</FileVersion>
<Version>2.4.5</Version>
<PackageTags>sync;rsync;synchronization</PackageTags>
<Description>
.NET library for file synchronization based on Rsync algorithm. Optimized for speed and data size to achieve best network performance.
You may also want to check [FastRsyncNet.Compression](https://www.nuget.org/packages/FastRsyncNet.Compression).
</Description>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Text.Json" Version="8.0.5"/>
<PackageReference Include="System.IO.Hashing" Version="8.0.0"/>
<PackageReference Include="System.Memory" Version="4.6.0" Condition="'$(TargetFramework)' != 'net7.0'"/>
</ItemGroup>
<ItemGroup>
<Folder Include="Properties\"/>
</ItemGroup>
</Project>
================================================
FILE: source/FastRsync/Hash/Adler32RollingChecksum.cs
================================================
using System;
namespace FastRsync.Hash
{
public class Adler32RollingChecksum : IRollingChecksum
{
public string Name => "Adler32";
public UInt32 Calculate(byte[] block, int offset, int count)
{
var a = 1;
var b = 0;
for (var i = offset; i < offset + count; i++)
{
var z = block[i];
a = (ushort)(z + a);
b = (ushort)(b + a);
}
return (UInt32) ((b << 16) | a);
}
public UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize)
{
var b = (ushort)(checksum >> 16 & 0xffff);
var a = (ushort)(checksum & 0xffff);
a = (ushort)((a - remove + add));
b = (ushort)((b - (chunkSize * remove) + a - 1));
return (UInt32) ((b << 16) | a);
}
}
}
================================================
FILE: source/FastRsync/Hash/Adler32RollingChecksumV2.cs
================================================
using System;
namespace FastRsync.Hash
{
[Obsolete("Adler32V2 has buggy mod operation implemented. See https://github.com/GrzegorzBlok/FastRsyncNet/issues/20")]
public class Adler32RollingChecksumV2 : IRollingChecksum
{
public string Name => "Adler32V2";
private const ushort Modulus = 65521;
public uint Calculate(byte[] block, int offset, int count)
{
var a = 1;
var b = 0;
for (var i = offset; i < offset + count; i++)
{
var z = block[i];
a = (z + a) % Modulus;
b = (b + a) % Modulus;
}
return (uint)((b << 16) | a);
}
public uint Rotate(uint checksum, byte remove, byte add, int chunkSize)
{
var b = (ushort)(checksum >> 16 & 0xffff);
var a = (ushort)(checksum & 0xffff);
a = (ushort)((a - remove + add) % Modulus);
b = (ushort)((b - (chunkSize * remove) + a - 1) % Modulus);
return (uint)((b << 16) | a);
}
}
}
================================================
FILE: source/FastRsync/Hash/Adler32RollingChecksumV3.cs
================================================
using System.Reflection;
using System.Runtime.CompilerServices;
namespace FastRsync.Hash
{
public class Adler32RollingChecksumV3 : IRollingChecksum
{
public string Name => "Adler32V3";
private const ushort Modulus = 65521;
public uint Calculate(byte[] block, int offset, int count)
{
var a = 1;
var b = 0;
for (count += offset; offset < count; ++offset)
{
a += block[offset];
a -= a >= Modulus ? Modulus : 0;
b += a;
b -= b >= Modulus ? Modulus : 0;
}
return (uint)((b << 16) | a);
}
public uint Rotate(uint checksum, byte remove, byte add, int chunkSize)
{
var b = (ushort)(checksum >> 16);
var a = (ushort)checksum;
var temp = a - remove + add;
a = (ushort)(temp < 0 ? temp + Modulus : temp >= Modulus ? temp - Modulus : temp);
temp = (b - (chunkSize * remove) + a - 1) % Modulus;
b = (ushort)(temp < 0 ? temp + Modulus : temp);
return (uint)((b << 16) | a);
}
}
}
================================================
FILE: source/FastRsync/Hash/CryptographyHashAlgorithmWrapper.cs
================================================
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace FastRsync.Hash
{
public class CryptographyHashAlgorithmWrapper : IHashAlgorithm
{
private readonly HashAlgorithm algorithm;
public CryptographyHashAlgorithmWrapper(string name, HashAlgorithm algorithm)
{
Name = name;
this.algorithm = algorithm;
}
public string Name { get; }
public int HashLengthInBytes => algorithm.HashSize / 8;
public byte[] ComputeHash(Stream stream)
{
return algorithm.ComputeHash(stream);
}
public async Task<byte[]> ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default)
{
#if (NET5_0_OR_GREATER)
return await algorithm.ComputeHashAsync(stream, cancellationToken).ConfigureAwait(false);
#else
return await Task.Run(() => algorithm.ComputeHash(stream), cancellationToken).ConfigureAwait(false);
#endif
}
public byte[] ComputeHash(byte[] buffer, int offset, int length)
{
return algorithm.ComputeHash(buffer, offset, length);
}
}
}
================================================
FILE: source/FastRsync/Hash/IHashAlgorithm.cs
================================================
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace FastRsync.Hash
{
public interface IHashAlgorithm
{
string Name { get; }
int HashLengthInBytes { get; }
byte[] ComputeHash(Stream stream);
Task<byte[]> ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default);
byte[] ComputeHash(byte[] buffer, int offset, int length);
}
}
================================================
FILE: source/FastRsync/Hash/IRollingChecksum.cs
================================================
using System;
namespace FastRsync.Hash
{
public interface IRollingChecksum
{
string Name { get; }
UInt32 Calculate(byte[] block, int offset, int count);
UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize);
}
}
================================================
FILE: source/FastRsync/Hash/NonCryptographicHashAlgorithmWrapper.cs
================================================
using System;
using System.IO.Hashing;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace FastRsync.Hash
{
public class NonCryptographicHashAlgorithmWrapper : IHashAlgorithm
{
public string Name { get; }
public int HashLengthInBytes => algorithm.HashLengthInBytes;
private readonly NonCryptographicHashAlgorithm algorithm;
public NonCryptographicHashAlgorithmWrapper(string name, NonCryptographicHashAlgorithm algorithm)
{
Name = name;
this.algorithm = algorithm;
}
public byte[] ComputeHash(Stream stream)
{
algorithm.Append(stream);
var hash = algorithm.GetHashAndReset();
Array.Reverse(hash);
return hash;
}
public async Task<byte[]> ComputeHashAsync(Stream stream, CancellationToken cancellationToken = default)
{
await algorithm.AppendAsync(stream, cancellationToken).ConfigureAwait(false);
var hash = algorithm.GetHashAndReset();
Array.Reverse(hash);
return hash;
}
public byte[] ComputeHash(byte[] buffer, int offset, int length)
{
var data = buffer.AsSpan(offset, length);
algorithm.Append(data);
var hash = algorithm.GetHashAndReset();
Array.Reverse(hash);
return hash;
}
}
}
================================================
FILE: source/FastRsync/Signature/ISignatureReader.cs
================================================
namespace FastRsync.Signature
{
public interface ISignatureReader
{
Signature ReadSignature();
Signature ReadSignatureMetadata();
}
}
================================================
FILE: source/FastRsync/Signature/ISignatureWriter.cs
================================================
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
namespace FastRsync.Signature
{
public interface ISignatureWriter
{
void WriteMetadata(SignatureMetadata metadata);
Task WriteMetadataAsync(SignatureMetadata metadata, CancellationToken cancellationToken);
void WriteChunk(ChunkSignature signature);
Task WriteChunkAsync(ChunkSignature signature, CancellationToken cancellationToken);
}
}
================================================
FILE: source/FastRsync/Signature/Signature.cs
================================================
using System.Collections.Generic;
using FastRsync.Core;
using FastRsync.Hash;
namespace FastRsync.Signature
{
public class SignatureMetadata
{
public string ChunkHashAlgorithm { get; set; }
public string RollingChecksumAlgorithm { get; set; }
public string BaseFileHashAlgorithm { get; set; }
public string BaseFileHash { get; set; }
}
public enum RsyncFormatType
{
Octodiff,
FastRsync
}
public class Signature
{
public Signature(SignatureMetadata metadata, RsyncFormatType type)
{
HashAlgorithm = SupportedAlgorithms.Hashing.Create(metadata.ChunkHashAlgorithm);
RollingChecksumAlgorithm = SupportedAlgorithms.Checksum.Create(metadata.RollingChecksumAlgorithm);
Chunks = new List<ChunkSignature>();
Metadata = metadata;
Type = type;
}
public IHashAlgorithm HashAlgorithm { get; private set; }
public IRollingChecksum RollingChecksumAlgorithm { get; private set; }
public List<ChunkSignature> Chunks { get; private set; }
public SignatureMetadata Metadata { get; private set; }
public RsyncFormatType Type { get; private set; }
}
}
================================================
FILE: source/FastRsync/Signature/SignatureBuilder.cs
================================================
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Diagnostics;
using FastRsync.Hash;
namespace FastRsync.Signature
{
public class SignatureBuilder
{
public const short MinimumChunkSize = 128;
public const short DefaultChunkSize = 2048;
public const short MaximumChunkSize = 31 * 1024;
private short chunkSize;
public SignatureBuilder() : this(SupportedAlgorithms.Hashing.Default(), SupportedAlgorithms.Checksum.Default())
{
}
public SignatureBuilder(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm)
{
HashAlgorithm = hashAlgorithm;
RollingChecksumAlgorithm = rollingChecksumAlgorithm;
ChunkSize = DefaultChunkSize;
ProgressReport = null;
}
public IProgress<ProgressReport> ProgressReport { get; set; }
public IHashAlgorithm HashAlgorithm { get; set; }
public IRollingChecksum RollingChecksumAlgorithm { get; set; }
public short ChunkSize
{
get => chunkSize;
set
{
if (value < MinimumChunkSize)
throw new ArgumentException($"Chunk size cannot be less than {MinimumChunkSize}");
if (value > MaximumChunkSize)
throw new ArgumentException($"Chunk size cannot be exceed {MaximumChunkSize}");
chunkSize = value;
}
}
public void Build(Stream baseDataStream, ISignatureWriter signatureWriter)
{
WriteMetadata(baseDataStream, signatureWriter);
WriteChunkSignatures(baseDataStream, signatureWriter);
}
public Task BuildAsync(Stream baseDataStream, ISignatureWriter signatureWriter) =>
BuildAsync(baseDataStream, signatureWriter, CancellationToken.None);
public async Task BuildAsync(Stream baseDataStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken)
{
await WriteMetadataAsync(baseDataStream, signatureWriter, cancellationToken).ConfigureAwait(false);
await WriteChunkSignaturesAsync(baseDataStream, signatureWriter, cancellationToken).ConfigureAwait(false);
}
private void WriteMetadata(Stream baseFileStream, ISignatureWriter signatureWriter)
{
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.HashingFile,
CurrentPosition = 0,
Total = baseFileStream.Length
});
baseFileStream.Seek(0, SeekOrigin.Begin);
var baseFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5();
var baseFileHash = baseFileVerificationHashAlgorithm.ComputeHash(baseFileStream);
signatureWriter.WriteMetadata(new SignatureMetadata
{
ChunkHashAlgorithm = HashAlgorithm.Name,
RollingChecksumAlgorithm = RollingChecksumAlgorithm.Name,
BaseFileHashAlgorithm = baseFileVerificationHashAlgorithm.Name,
BaseFileHash = Convert.ToBase64String(baseFileHash)
});
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.HashingFile,
CurrentPosition = baseFileStream.Length,
Total = baseFileStream.Length
});
}
private async Task WriteMetadataAsync(Stream baseFileStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken)
{
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.HashingFile,
CurrentPosition = 0,
Total = baseFileStream.Length
});
baseFileStream.Seek(0, SeekOrigin.Begin);
var baseFileVerificationHashAlgorithm = SupportedAlgorithms.Hashing.Md5();
var baseFileHash = await baseFileVerificationHashAlgorithm.ComputeHashAsync(baseFileStream, cancellationToken).ConfigureAwait(false);
await signatureWriter.WriteMetadataAsync(new SignatureMetadata
{
ChunkHashAlgorithm = HashAlgorithm.Name,
RollingChecksumAlgorithm = RollingChecksumAlgorithm.Name,
BaseFileHashAlgorithm = baseFileVerificationHashAlgorithm.Name,
BaseFileHash = Convert.ToBase64String(baseFileHash)
}, cancellationToken).ConfigureAwait(false);
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.HashingFile,
CurrentPosition = baseFileStream.Length,
Total = baseFileStream.Length
});
}
private void WriteChunkSignatures(Stream baseFileStream, ISignatureWriter signatureWriter)
{
var checksumAlgorithm = RollingChecksumAlgorithm;
var hashAlgorithm = HashAlgorithm;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingSignatures,
CurrentPosition = 0,
Total = baseFileStream.Length
});
baseFileStream.Seek(0, SeekOrigin.Begin);
long start = 0;
int read;
var block = new byte[ChunkSize];
while ((read = baseFileStream.Read(block, 0, block.Length)) > 0)
{
signatureWriter.WriteChunk(new ChunkSignature
{
StartOffset = start,
Length = (short)read,
Hash = hashAlgorithm.ComputeHash(block, 0, read),
RollingChecksum = checksumAlgorithm.Calculate(block, 0, read)
});
start += read;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingSignatures,
CurrentPosition = start,
Total = baseFileStream.Length
});
}
}
private async Task WriteChunkSignaturesAsync(Stream baseFileStream, ISignatureWriter signatureWriter, CancellationToken cancellationToken)
{
var checksumAlgorithm = RollingChecksumAlgorithm;
var hashAlgorithm = HashAlgorithm;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingSignatures,
CurrentPosition = 0,
Total = baseFileStream.Length
});
baseFileStream.Seek(0, SeekOrigin.Begin);
long start = 0;
int read;
var block = new byte[ChunkSize];
while ((read = await baseFileStream.ReadAsync(block, 0, block.Length, cancellationToken).ConfigureAwait(false)) > 0)
{
await signatureWriter.WriteChunkAsync(new ChunkSignature
{
StartOffset = start,
Length = (short)read,
Hash = hashAlgorithm.ComputeHash(block, 0, read),
RollingChecksum = checksumAlgorithm.Calculate(block, 0, read)
}, cancellationToken).ConfigureAwait(false);
start += read;
ProgressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.BuildingSignatures,
CurrentPosition = start,
Total = baseFileStream.Length
});
}
}
}
}
================================================
FILE: source/FastRsync/Signature/SignatureReader.cs
================================================
using System;
using System.Collections;
using System.IO;
using System.Text.Json;
using FastRsync.Core;
using FastRsync.Diagnostics;
namespace FastRsync.Signature
{
public class SignatureReader : ISignatureReader
{
private readonly IProgress<ProgressReport> report;
private readonly BinaryReader reader;
public SignatureReader(Stream stream, IProgress<ProgressReport> progressHandler)
{
this.report = progressHandler;
this.reader = new BinaryReader(stream);
}
public Signature ReadSignature()
{
Progress();
var signature = ReadSignatureMetadata();
ReadChunks(signature);
return signature;
}
public Signature ReadSignatureMetadata()
{
var header = reader.ReadBytes(BinaryFormat.SignatureFormatHeaderLength);
if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.SignatureHeader, header))
{
return ReadFastRsyncSignatureHeader();
}
if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.SignatureHeader, header))
{
return ReadOctoSignatureHeader();
}
throw new InvalidDataException(
"The signature file uses a different file format than this program can handle.");
}
private Signature ReadFastRsyncSignatureHeader()
{
var version = reader.ReadByte();
if (version != FastRsyncBinaryFormat.Version)
throw new InvalidDataException(
"The signature file uses a newer file format than this program can handle.");
var metadataStr = reader.ReadString();
#if NET7_0_OR_GREATER
var metadata = JsonSerializer.Deserialize(metadataStr, JsonContextCore.Default.SignatureMetadata);
#else
var metadata =
JsonSerializer.Deserialize<SignatureMetadata>(metadataStr, JsonSerializationSettings.JsonSettings);
#endif
var signature = new Signature(metadata, RsyncFormatType.FastRsync);
return signature;
}
private Signature ReadOctoSignatureHeader()
{
var version = reader.ReadByte();
if (version != OctoBinaryFormat.Version)
throw new InvalidDataException(
"The signature file uses a newer file format than this program can handle.");
var hashAlgorithmName = reader.ReadString();
var rollingChecksumAlgorithmName = reader.ReadString();
var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length);
if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta))
throw new InvalidDataException("The signature file appears to be corrupt.");
Progress();
var hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName);
var rollingChecksumAlgorithm = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithmName);
var signature = new Signature(new SignatureMetadata
{
ChunkHashAlgorithm = hashAlgorithm.Name,
RollingChecksumAlgorithm = rollingChecksumAlgorithm.Name
}, RsyncFormatType.Octodiff);
return signature;
}
private void ReadChunks(Signature signature)
{
var expectedHashLength = signature.HashAlgorithm.HashLengthInBytes;
long start = 0;
var signatureLength = reader.BaseStream.Length;
var remainingBytes = signatureLength - reader.BaseStream.Position;
var signatureSize = sizeof(ushort) + sizeof(uint) + expectedHashLength;
if (remainingBytes % signatureSize != 0)
throw new InvalidDataException(
"The signature file appears to be corrupt; at least one chunk has data missing.");
while (reader.BaseStream.Position < signatureLength - 1)
{
var length = reader.ReadInt16();
var checksum = reader.ReadUInt32();
var chunkHash = reader.ReadBytes(expectedHashLength);
signature.Chunks.Add(new ChunkSignature
{
StartOffset = start,
Length = length,
RollingChecksum = checksum,
Hash = chunkHash
});
start += length;
Progress();
}
}
private void Progress()
{
report?.Report(new ProgressReport
{
Operation = ProgressOperationType.ReadingSignature,
CurrentPosition = reader.BaseStream.Position,
Total = reader.BaseStream.Length
});
}
}
}
================================================
FILE: source/FastRsync/Signature/SignatureWriter.cs
================================================
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Core;
namespace FastRsync.Signature
{
public class SignatureWriter : ISignatureWriter
{
private readonly Stream signatureStream;
private readonly BinaryWriter signaturebw;
public SignatureWriter(Stream signatureStream)
{
this.signatureStream = signatureStream;
signaturebw = new BinaryWriter(signatureStream);
}
private static void WriteMetadataInternal(BinaryWriter bw, SignatureMetadata metadata)
{
bw.Write(FastRsyncBinaryFormat.SignatureHeader);
bw.Write(FastRsyncBinaryFormat.Version);
#if NET7_0_OR_GREATER
var metadataStr = JsonSerializer.Serialize(metadata, JsonContextCore.Default.SignatureMetadata);
#else
var metadataStr = JsonSerializer.Serialize(metadata, JsonSerializationSettings.JsonSettings);
#endif
bw.Write(metadataStr);
}
public void WriteMetadata(SignatureMetadata metadata)
{
WriteMetadataInternal(signaturebw, metadata);
}
public async Task WriteMetadataAsync(SignatureMetadata metadata, CancellationToken cancellationToken)
{
var ms = new MemoryStream(256);
var msbw = new BinaryWriter(ms);
WriteMetadataInternal(msbw, metadata);
ms.Seek(0, SeekOrigin.Begin);
#if (NET5_0_OR_GREATER)
await ms.CopyToAsync(signatureStream, cancellationToken).ConfigureAwait(false);
#else
await ms.CopyToAsync(signatureStream).ConfigureAwait(false);
#endif
}
public void WriteChunk(ChunkSignature signature)
{
signaturebw.Write(signature.Length);
signaturebw.Write(signature.RollingChecksum);
signaturebw.Write(signature.Hash);
}
public async Task WriteChunkAsync(ChunkSignature signature, CancellationToken cancellationToken)
{
signaturebw.Write(signature.Length);
signaturebw.Write(signature.RollingChecksum);
await signatureStream.WriteAsync(signature.Hash, 0, signature.Hash.Length, cancellationToken)
.ConfigureAwait(false);
}
}
}
================================================
FILE: source/FastRsync.BackwardCompatibilityTests/BackwardCompatibilityTests.cs
================================================
using System;
using System.IO;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
using FastRsync.Signature;
using NSubstitute;
using NUnit.Framework;
using NUnit.Framework.Legacy;
namespace FastRsync.BackwardCompatibilityTests
{
[TestFixture]
public class BackwardCompatibilityTests
{
[Test]
[TestCase(1378, 129)]
[TestCase(13780, 1290)]
[TestCase(137800, 1290)]
public async Task LegacyLibraryAppliesNewPatch(int baseNumberOfBytes, int newDataNumberOfBytes)
{
// Arrange - signature and patch from the current library
var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false);
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
// Act using the legacy library
var progressReporter = Substitute.For<IProgress<FastRsyncLegacy231.Diagnostics.ProgressReport>>();
var patchedDataStream = new MemoryStream();
var deltaApplier = new FastRsyncLegacy231.Delta.DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new FastRsyncLegacy231.Delta.BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<FastRsyncLegacy231.Diagnostics.ProgressReport>());
}
[Test]
[TestCase(1378, 129)]
[TestCase(13780, 1290)]
[TestCase(137800, 1290)]
public async Task LegacyLibraryPreparesAndAppliesNewPatch(int baseNumberOfBytes, int newDataNumberOfBytes)
{
// Arrange - signature from the current library
var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false);
// Act using the legacy library
var deltaStream = new MemoryStream();
var deltaBuilder = new FastRsyncLegacy231.Delta.DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new FastRsyncLegacy231.Signature.SignatureReader(baseSignatureStream, null),
new FastRsyncLegacy231.Core.AggregateCopyOperationsDecorator(new FastRsyncLegacy231.Delta.BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
var progressReporter = Substitute.For<IProgress<FastRsyncLegacy231.Diagnostics.ProgressReport>>();
var patchedDataStream = new MemoryStream();
var deltaApplier = new FastRsyncLegacy231.Delta.DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new FastRsyncLegacy231.Delta.BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<FastRsyncLegacy231.Diagnostics.ProgressReport>());
}
[Test]
[TestCase(1378, 129)]
[TestCase(13780, 1290)]
[TestCase(137800, 1290)]
public async Task LegacyLibraryPreparesPatchForNewLibraryToApplyIt(int baseNumberOfBytes, int newDataNumberOfBytes)
{
// Arrange - signature from current library, patch from old library
var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, SupportedAlgorithms.Checksum.Adler32Rolling()).ConfigureAwait(false);
var deltaStream = new MemoryStream();
var deltaBuilder = new FastRsyncLegacy231.Delta.DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new FastRsyncLegacy231.Signature.SignatureReader(baseSignatureStream, null),
new FastRsyncLegacy231.Core.AggregateCopyOperationsDecorator(new FastRsyncLegacy231.Delta.BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
// Act - apply patch using the current library
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
private static async Task<(MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream)> PrepareTestDataAsync(int baseNumberOfBytes, int newDataNumberOfBytes, Hash.IRollingChecksum rollingChecksumAlg)
{
var baseData = new byte[baseNumberOfBytes];
new Random().NextBytes(baseData);
var baseDataStream = new MemoryStream(baseData);
var baseSignatureStream = new MemoryStream();
var signatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), rollingChecksumAlg);
await signatureBuilder.BuildAsync(baseDataStream, new SignatureWriter(baseSignatureStream)).ConfigureAwait(false);
baseSignatureStream.Seek(0, SeekOrigin.Begin);
var newData = new byte[newDataNumberOfBytes];
new Random().NextBytes(newData);
var newDataStream = new MemoryStream(newData);
return (baseDataStream, baseSignatureStream, newData, newDataStream);
}
}
}
================================================
FILE: source/FastRsync.BackwardCompatibilityTests/FastRsync.BackwardCompatibilityTests.csproj
================================================
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net472</TargetFramework>
<LangVersion>7.3</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="NUnit" Version="4.4.0" />
<PackageReference Include="NUnit3TestAdapter" Version="6.1.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
<PackageReference Include="System.Data.HashFunction.Core" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.Interfaces" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FastRsync\FastRsync.csproj" />
</ItemGroup>
<ItemGroup>
<Reference Include="FastRsyncLegacy231">
<HintPath>TestData\FastRsyncLegacy231.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<None Update="TestData\FastRsyncLegacy231.dll">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
================================================
FILE: source/FastRsync.Benchmarks/BuildPatchBenchmark.cs
================================================
using System;
using System.IO;
using BenchmarkDotNet.Attributes;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Signature;
namespace FastRsync.Benchmarks
{
public class BuildPatchBenchmark
{
[Params(128, 16974)]
public int BaseFileSize { get; set; }
[Params(16974, 128)]
public int NewFileSize { get; set; }
private byte[] newFileData;
private readonly DeltaBuilder deltaBuilder = new();
private MemoryStream newDataStream;
private MemoryStream baseSignatureSha1Stream;
private MemoryStream baseSignatureXxHashStream;
private MemoryStream baseSignatureXxHash3Stream;
private MemoryStream baseSignatureMd5Stream;
[GlobalSetup]
public void GlobalSetup()
{
var baseFileBytes = new byte[BaseFileSize];
var rnd = new Random();
rnd.NextBytes(baseFileBytes);
newFileData = new byte[NewFileSize];
rnd.NextBytes(newFileData);
var baseDataStream = new MemoryStream(baseFileBytes);
newDataStream = new MemoryStream(newFileData);
{
var xxHashSignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(),
SupportedAlgorithms.Checksum.Adler32Rolling());
baseSignatureXxHashStream = new MemoryStream();
xxHashSignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureXxHashStream));
}
{
var xxHash3SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.XxHash3(),
SupportedAlgorithms.Checksum.Adler32Rolling());
baseSignatureXxHash3Stream = new MemoryStream();
xxHash3SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureXxHashStream));
}
{
var sha1SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.Sha1(),
SupportedAlgorithms.Checksum.Adler32Rolling());
baseDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureSha1Stream = new MemoryStream();
sha1SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureSha1Stream));
}
{
var md5SignatureBuilder = new SignatureBuilder(SupportedAlgorithms.Hashing.Md5(),
SupportedAlgorithms.Checksum.Adler32Rolling());
baseDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureMd5Stream = new MemoryStream();
md5SignatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureMd5Stream));
}
}
[Benchmark]
public byte[] BuildPatchXxHash()
{
newDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureXxHashStream.Seek(0, SeekOrigin.Begin);
var deltaStream = new MemoryStream();
deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureXxHashStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
return deltaStream.ToArray();
}
[Benchmark]
public byte[] BuildPatchXxHash3()
{
newDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureXxHash3Stream.Seek(0, SeekOrigin.Begin);
var deltaStream = new MemoryStream();
deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureXxHash3Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
return deltaStream.ToArray();
}
[Benchmark]
public byte[] BuildPatchSha1()
{
newDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureSha1Stream.Seek(0, SeekOrigin.Begin);
var deltaStream = new MemoryStream();
deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureSha1Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
return deltaStream.ToArray();
}
[Benchmark]
public byte[] BuildPatchMd5()
{
newDataStream.Seek(0, SeekOrigin.Begin);
baseSignatureMd5Stream.Seek(0, SeekOrigin.Begin);
var deltaStream = new MemoryStream();
deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureMd5Stream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
return deltaStream.ToArray();
}
}
}
================================================
FILE: source/FastRsync.Benchmarks/FastRsync.Benchmarks.csproj
================================================
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="12.27.0" />
<PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
<PackageReference Include="System.IO.Hashing" Version="10.0.1" />
<PackageReference Include="System.Data.HashFunction.Core" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.Interfaces" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FastRsync.Tests\FastRsync.Tests.csproj" />
<ProjectReference Include="..\FastRsync\FastRsync.csproj" />
</ItemGroup>
</Project>
================================================
FILE: source/FastRsync.Benchmarks/HashBenchmark.cs
================================================
using System;
using System.Data.HashFunction.xxHash;
using System.IO;
using System.IO.Hashing;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Specialized;
using BenchmarkDotNet.Attributes;
namespace FastRsync.Benchmarks
{
public class HashBenchmark
{
[Params(128, 16974, 356879)]
public int N { get; set; }
private byte[] data;
private MemoryStream streamdata;
private const string AzureBlobStorageConnectionString = "PUT AZURE BLOB STORAGE CONNECTION STRING HERE";
private readonly BlobServiceClient azureBlobServiceClient = new(AzureBlobStorageConnectionString);
private BlobContainerClient azureBlobContainerClient;
private readonly IxxHash dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig
{
HashSizeInBits = 64
});
private readonly NonCryptographicHashAlgorithm systemXxHash = new XxHash64();
[GlobalSetup]
public async Task GlobalSetup()
{
data = new byte[N];
new Random().NextBytes(data);
var sdata = new byte[N];
new Random().NextBytes(sdata);
streamdata = new MemoryStream(sdata);
azureBlobContainerClient = azureBlobServiceClient.GetBlobContainerClient("hashbenchmark");
BlockBlobClient azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata");
await azureBlob.UploadAsync(streamdata);
}
[Benchmark]
public byte[] DataXxHash64()
{
return dataXxHash.ComputeHash(data).Hash;
}
[Benchmark]
public byte[] DataXxHash64MemoryStream()
{
streamdata.Seek(0, SeekOrigin.Begin);
return dataXxHash.ComputeHash(streamdata).Hash;
}
[Benchmark]
public byte[] DataXxHash64AzureBlobStream()
{
var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata");
using var blobstream = azureBlob.OpenRead();
return dataXxHash.ComputeHash(blobstream).Hash;
}
[Benchmark]
public byte[] SystemXxHash64()
{
return XxHash64.Hash(data);
}
[Benchmark]
public byte[] SystemXxHash64MemoryStream()
{
streamdata.Seek(0, SeekOrigin.Begin);
systemXxHash.Append(streamdata);
return systemXxHash.GetHashAndReset();
}
[Benchmark]
public byte[] SystemXxHash64AzureBlobStream()
{
var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata");
using var blobstream = azureBlob.OpenRead();
systemXxHash.Append(blobstream);
return systemXxHash.GetHashAndReset();
}
[Benchmark]
public byte[] SystemXxHash64Reverse()
{
var systemXxHash64Dest = XxHash64.Hash(data);
Array.Reverse(systemXxHash64Dest);
return systemXxHash64Dest;
}
[Benchmark]
public byte[] SystemXxHash64AzureBlobStreamReverse()
{
var azureBlob = azureBlobContainerClient.GetBlockBlobClient("hashdata");
using var blobstream = azureBlob.OpenRead();
systemXxHash.Append(blobstream);
var systemXxHash64Dest = systemXxHash.GetHashAndReset();
Array.Reverse(systemXxHash64Dest);
return systemXxHash64Dest;
}
}
}
================================================
FILE: source/FastRsync.Benchmarks/Program.cs
================================================
using System.Reflection;
using BenchmarkDotNet.Running;
namespace FastRsync.Benchmarks
{
class Program
{
static void Main(string[] args)
{
BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args);
}
}
}
================================================
FILE: source/FastRsync.Benchmarks/RollingChecksumBenchmark.cs
================================================
using System;
using BenchmarkDotNet.Attributes;
using FastRsync.Core;
using FastRsync.Hash;
namespace FastRsync.Benchmarks
{
public class RollingCheckSumBenchmark
{
[Params(128, 16974, 356879)]
public int N { get; set; }
private byte[] data;
private readonly IRollingChecksum adler32RollingAlgorithm = SupportedAlgorithms.Checksum.Adler32Rolling();
private readonly IRollingChecksum adler32RollingV2Algorithm = SupportedAlgorithms.Checksum.Adler32RollingV2();
private readonly IRollingChecksum adler32RollingV3Algorithm = SupportedAlgorithms.Checksum.Adler32RollingV3();
private uint checksum32;
private uint checksum32V2;
private uint checksum32V3;
[GlobalSetup]
public void GlobalSetup()
{
data = new byte[N];
new Random().NextBytes(data);
checksum32 = adler32RollingAlgorithm.Calculate(data, 0, data.Length - 1);
checksum32V2 = adler32RollingV2Algorithm.Calculate(data, 0, data.Length - 1);
checksum32V3 = adler32RollingV3Algorithm.Calculate(data, 0, data.Length - 1);
}
[Benchmark]
public uint Adler32RollingCalculateChecksum()
{
return adler32RollingAlgorithm.Calculate(data, 0, data.Length);
}
[Benchmark]
public uint Adler32RollingV2CalculateChecksum()
{
return adler32RollingV2Algorithm.Calculate(data, 0, data.Length);
}
[Benchmark]
public uint Adler32RollingV3CalculateChecksum()
{
return adler32RollingV3Algorithm.Calculate(data, 0, data.Length);
}
[Benchmark]
public uint Adler32RollingRotateChecksum()
{
return adler32RollingAlgorithm.Rotate(checksum32, data[0], data[^1], data.Length - 1);
}
[Benchmark]
public uint Adler32RollingV2RotateChecksum()
{
return adler32RollingV2Algorithm.Rotate(checksum32V2, data[0], data[^1], data.Length - 1);
}
[Benchmark]
public uint Adler32RollingV3RotateChecksum()
{
return adler32RollingV3Algorithm.Rotate(checksum32V3, data[0], data[^1], data.Length - 1);
}
}
}
================================================
FILE: source/FastRsync.Benchmarks/SignatureBenchmark.cs
================================================
using System;
using System.IO;
using BenchmarkDotNet.Attributes;
using FastRsync.Core;
using FastRsync.Signature;
using FastRsync.Tests.OctodiffLegacy;
namespace FastRsync.Benchmarks
{
public class SignatureBenchmark
{
[Params(128, 16974, 356879)]
public int BaseFileSize { get; set; }
[Params(SignatureBuilder.MinimumChunkSize, SignatureBuilder.DefaultChunkSize, SignatureBuilder.MaximumChunkSize)]
public short ChunkSize { get; set; }
private byte[] data;
private readonly SignatureBuilder xxHashSignatureBuilder =
new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32Rolling());
private readonly SignatureBuilder xxHash3SignatureBuilder =
new(SupportedAlgorithms.Hashing.XxHash3(), SupportedAlgorithms.Checksum.Adler32Rolling());
private readonly SignatureBuilder xxHashAdler32V2SignatureBuilder =
new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32RollingV2());
private readonly SignatureBuilder sha1SignatureBuilder =
new(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Adler32Rolling());
private readonly SignatureBuilder md5SignatureBuilder =
new(SupportedAlgorithms.Hashing.Md5(), SupportedAlgorithms.Checksum.Adler32Rolling());
private readonly OctodiffSignatureBuilder xxHashOctodiffSignatureBuilder =
new(SupportedAlgorithms.Hashing.XxHash(), SupportedAlgorithms.Checksum.Adler32Rolling());
private MemoryStream dataStream;
[GlobalSetup]
public void GlobalSetup()
{
data = new byte[BaseFileSize];
new Random().NextBytes(data);
xxHashSignatureBuilder.ChunkSize = ChunkSize;
xxHashAdler32V2SignatureBuilder.ChunkSize = ChunkSize;
sha1SignatureBuilder.ChunkSize = ChunkSize;
md5SignatureBuilder.ChunkSize = ChunkSize;
dataStream = new MemoryStream(data);
}
[Benchmark]
public byte[] SignaturexxHash()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
xxHashSignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream));
return signatureStream.ToArray();
}
[Benchmark]
public byte[] SignaturexxHashAdler32V2()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
xxHashAdler32V2SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream));
return signatureStream.ToArray();
}
[Benchmark]
public byte[] SignaturexxHash3()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
xxHash3SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream));
return signatureStream.ToArray();
}
[Benchmark]
public byte[] SignatureSha1()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
sha1SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream));
return signatureStream.ToArray();
}
[Benchmark]
public byte[] SignatureMd5()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
md5SignatureBuilder.Build(dataStream, new SignatureWriter(signatureStream));
return signatureStream.ToArray();
}
[Benchmark]
public byte[] OctodiffSignaturexxHash()
{
dataStream.Seek(0, SeekOrigin.Begin);
var signatureStream = new MemoryStream();
xxHashOctodiffSignatureBuilder.Build(dataStream, new OctodiffSignatureWriter(signatureStream));
return signatureStream.ToArray();
}
}
}
================================================
FILE: source/FastRsync.Compression/FastRsync.Compression.csproj
================================================
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Authors>Grzegorz Blok</Authors>
<Company />
<PackageId>FastRsyncNet.Compression</PackageId>
<Product></Product>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<Copyright></Copyright>
<Description>
.NET library for file compression compatible with Rsync algorithm.
You may also want to check [FastRsyncNet](https://www.nuget.org/packages/FastRsyncNet).
</Description>
<PackageProjectUrl>https://github.com/GrzegorzBlok/FastRsyncNet</PackageProjectUrl>
<PackageTags>sync;rsync;synchronization;compression</PackageTags>
<AssemblyVersion>1.1.2</AssemblyVersion>
<FileVersion>1.1.2</FileVersion>
<Version>1.1.2</Version>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ProDotNetZip" Version="1.20.0" />
</ItemGroup>
</Project>
================================================
FILE: source/FastRsync.Compression/GZip.cs
================================================
// this implementation is influenced by pigz tool by Mark Adler: https://github.com/madler/pigz/blob/master/pigz.c
// pigz license:
/*
This software is provided 'as-is', without any express or implied
warranty. In no event will the author be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Mark Adler
madler@alumni.caltech.edu
*/
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Ionic.Zlib;
namespace FastRsync.Compression
{
public class GZip
{
const int RSYNCBITS = 12;
const uint RSYNCMASK = ((1U << RSYNCBITS) - (uint)1);
const uint RSYNCHIT = (RSYNCMASK >> 1);
const int BUFFER_SIZE = 32 * 1024;
public static void Compress(Stream sourceStream, Stream destStream)
{
var buffer = new byte[BUFFER_SIZE];
using (var compressor = new GZipStream(destStream, Ionic.Zlib.CompressionMode.Compress,
CompressionLevel.BestSpeed, true))
{
compressor.LastModified = new DateTime(1970, 1, 1);
uint hash = RSYNCHIT;
int n;
while ((n = sourceStream.Read(buffer, 0, BUFFER_SIZE)) > 0)
{
for (int i = 0, j = 0; i < n; i++)
{
hash = ((hash << 1) ^ buffer[i]) & RSYNCMASK;
if (hash == RSYNCHIT)
{
compressor.FlushMode = FlushType.Sync;
compressor.Write(buffer, j, i + 1 - j);
j = i + 1;
}
else if (i + 1 == n)
{
compressor.FlushMode = FlushType.None;
compressor.Write(buffer, j, i + 1 - j);
}
}
}
}
}
public static Task CompressAsync(Stream sourceStream, Stream destStream) =>
CompressAsync(sourceStream, destStream, CancellationToken.None);
public static async Task CompressAsync(Stream sourceStream, Stream destStream, CancellationToken cancellationToken)
{
var buffer = new byte[BUFFER_SIZE];
using (var compressor = new GZipStream(destStream, Ionic.Zlib.CompressionMode.Compress,
CompressionLevel.BestSpeed, true))
{
compressor.LastModified = new DateTime(1970, 1, 1);
uint hash = RSYNCHIT;
int n;
while ((n = await sourceStream.ReadAsync(buffer, 0, BUFFER_SIZE, cancellationToken).ConfigureAwait(false)) > 0)
{
for (int i = 0, j = 0; i < n; i++)
{
hash = ((hash << 1) ^ buffer[i]) & RSYNCMASK;
if (hash == RSYNCHIT)
{
compressor.FlushMode = FlushType.Sync;
await compressor.WriteAsync(buffer, j, i + 1 - j, cancellationToken).ConfigureAwait(false);
j = i + 1;
}
else if (i + 1 == n)
{
compressor.FlushMode = FlushType.None;
await compressor.WriteAsync(buffer, j, i + 1 - j, cancellationToken).ConfigureAwait(false);
}
}
}
}
}
}
}
================================================
FILE: source/FastRsync.Tests/Adler32RollingChecksumAlgorithmsTests.cs
================================================
using FastRsync.Hash;
using NSubstitute;
using NUnit.Framework;
using NUnit.Framework.Legacy;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
namespace FastRsync.Tests;
[TestFixture]
public class Adler32RollingChecksumAlgorithmsTests
{
public static IRollingChecksum[] RollingChecksumAlgorithms =>
typeof(IRollingChecksum).Assembly.GetTypes()
.Where(t => typeof(IRollingChecksum).IsAssignableFrom(t)
&& t is {IsClass: true, IsAbstract:false}
&& t.GetConstructor(Type.EmptyTypes) != null
&& t.GetCustomAttribute<ObsoleteAttribute>() == null)
.Select(t => (IRollingChecksum)Activator.CreateInstance(t))
.ToArray();
private static readonly int[] Sizes = [1024 * 1024 / 2, 10 * 1024 * 1024, 20 * 1024 * 1024];
private static readonly int[] ModificationPercentages = [1, 3, 5, 10];
[Test]
[TestCaseSource(nameof(RollingChecksumAlgorithms))]
public void EveryRollingChecksumAlgorithmCalculatesRotateCorrectly(IRollingChecksum rollingChecksumAlgorithmInstance)
{
// Arrange
const int len = 8 * 1024;
const int windowSize = len - 1;
var bytes = new byte[len];
new Random().NextBytes(bytes);
// Act
var rotatedChecksum = rollingChecksumAlgorithmInstance.Rotate(
rollingChecksumAlgorithmInstance.Calculate(bytes, 0, windowSize),
bytes[0],
bytes[^1],
windowSize);
var checksum = rollingChecksumAlgorithmInstance.Calculate(bytes, 1, windowSize);
// Assert
Assert.That(rotatedChecksum, Is.EqualTo(checksum));
}
public static IEnumerable<TestCaseData> SignDeltaAndApplyPatchTestCases()
{
return
from size in Sizes
from modPercentage in ModificationPercentages
from algo in RollingChecksumAlgorithms
select new TestCaseData(size, modPercentage, algo);
}
[Test]
[TestCaseSource(nameof(SignDeltaAndApplyPatchTestCases))]
public async Task SignDeltaAndApplyPatchWithEveryRollingChecksumAlgorithm(int size, int modificationPercentage, IRollingChecksum algorithm)
{
//Arrange
var bytesToPatchFrom = GetRandomBytes(size);
var targetBytes = ModifyBytes(bytesToPatchFrom, modificationPercentage);
//Act
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
//generate signature of target bytes
var signatureStream = new MemoryStream();
var signatureBuilder = new Signature.SignatureBuilder(SupportedAlgorithms.Hashing.XxHash(), algorithm);
await signatureBuilder.BuildAsync(new MemoryStream(bytesToPatchFrom), new Signature.SignatureWriter(signatureStream));
signatureStream.Seek(0, SeekOrigin.Begin);
//generate delta of base bytes and target bytes
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(new MemoryStream(targetBytes), new Signature.SignatureReader(signatureStream, progressReporter), new BinaryDeltaWriter(deltaStream));
deltaStream.Seek(0, SeekOrigin.Begin);
//apply delta to base bytes
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(new MemoryStream(bytesToPatchFrom), new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream);
patchedDataStream.Seek(0, SeekOrigin.Begin);
//Assert
CollectionAssert.AreEqual(targetBytes, patchedDataStream.ToArray());
//make sure that the delta is reasonably small, allowing for some overhead but ensuring that the rolling checksum is effective
Assert.That(deltaStream.Length, Is.LessThanOrEqualTo(2.5 * modificationPercentage / 100.0 * bytesToPatchFrom.Length));
}
private static byte[] ModifyBytes(ReadOnlySpan<byte> input, int percentage)
{
Assert.That(percentage, Is.InRange(0, 100));
var length = input.Length;
var modified = new byte[length];
input.CopyTo(modified);
var bytesToModify = (int)(length * (percentage / 100.0));
if (bytesToModify == 0) return modified;
var random = new Random();
var maxStartPosition = length - bytesToModify;
var startIndex = maxStartPosition > 0 ? random.Next(maxStartPosition + 1) : 0;
// Modify bytes in a continuous range
for (var i = startIndex; i < startIndex + bytesToModify && i < length; i++)
{
modified[i] = (byte)(modified[i] + 1);
}
return modified;
}
private static byte[] GetRandomBytes(int size)
{
var data = new byte[size];
new Random().NextBytes(data);
return data;
}
}
================================================
FILE: source/FastRsync.Tests/Adler32RollingChecksumTests.cs
================================================
using FastRsync.Hash;
using NUnit.Framework;
namespace FastRsync.Tests;
[TestFixture]
public class Adler32RollingChecksumTests
{
[Test]
public void Adler32RollingChecksum_CalculatesChecksum()
{
// Arrange
var data1 = "Adler32 checksum test"u8.ToArray();
var data2 = "Fast Rsync Fast Rsync"u8.ToArray();
var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray();
var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray();
var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray();
// Act
var checksum1 = new Adler32RollingChecksum().Calculate(data1, 0, data1.Length);
var checksum2 = new Adler32RollingChecksum().Calculate(data2, 0, data2.Length);
var checksum3 = new Adler32RollingChecksum().Calculate(data3, 0, data3.Length);
var checksum4 = new Adler32RollingChecksum().Calculate(data4, 0, data4.Length);
var checksum5 = new Adler32RollingChecksum().Calculate(data5, 0, data5.Length);
// Assert
Assert.That(checksum1, Is.EqualTo(0x4ff907a1));
Assert.That(checksum2, Is.EqualTo(0x5206079b));
//Assert.That(checksum3, Is.EqualTo(0x040f0fc1)); // bug in adler32 implementation https://github.com/OctopusDeploy/Octodiff/issues/16
//Assert.That(checksum4, Is.EqualTo(0x2d10357d));
//Assert.That(checksum5, Is.EqualTo(0xa05ca509));
}
}
================================================
FILE: source/FastRsync.Tests/Adler32RollingChecksumV2Tests.cs
================================================
using FastRsync.Hash;
using NUnit.Framework;
namespace FastRsync.Tests;
[TestFixture]
public class Adler32RollingChecksumV2Tests
{
[Test]
public void Adler32RollingChecksumV2_CalculatesChecksum()
{
// Arrange
var data1 = "Adler32 checksum test"u8.ToArray();
var data2 = "Fast Rsync Fast Rsync"u8.ToArray();
var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray();
var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray();
var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray();
// Act
var checksum1 = new Adler32RollingChecksumV2().Calculate(data1, 0, data1.Length);
var checksum2 = new Adler32RollingChecksumV2().Calculate(data2, 0, data2.Length);
var checksum3 = new Adler32RollingChecksumV2().Calculate(data3, 0, data3.Length);
var checksum4 = new Adler32RollingChecksumV2().Calculate(data4, 0, data4.Length);
var checksum5 = new Adler32RollingChecksumV2().Calculate(data5, 0, data5.Length);
// Assert
Assert.That(checksum1, Is.EqualTo(0x4ff907a1));
Assert.That(checksum2, Is.EqualTo(0x5206079b));
Assert.That(checksum3, Is.EqualTo(0x040f0fc1));
Assert.That(checksum4, Is.EqualTo(0x2d10357d));
Assert.That(checksum5, Is.EqualTo(0xa05ca509));
}
}
================================================
FILE: source/FastRsync.Tests/Adler32RollingChecksumV3Tests.cs
================================================
using FastRsync.Hash;
using NUnit.Framework;
using System;
namespace FastRsync.Tests;
[TestFixture]
public class Adler32RollingChecksumV3Tests
{
[Test]
public void Adler32RollingChecksumV3_CalculatesChecksum()
{
// Arrange
var data1 = "Adler32 checksum test"u8.ToArray();
var data2 = "Fast Rsync Fast Rsync"u8.ToArray();
var data3 = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"u8.ToArray();
var data4 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis malesuada turpis non libero faucibus sodales. Mauris eget justo est. Pellentesque."u8.ToArray();
var data5 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."u8.ToArray();
// Act
var checksum1 = new Adler32RollingChecksumV3().Calculate(data1, 0, data1.Length);
var checksum2 = new Adler32RollingChecksumV3().Calculate(data2, 0, data2.Length);
var checksum3 = new Adler32RollingChecksumV3().Calculate(data3, 0, data3.Length);
var checksum4 = new Adler32RollingChecksumV3().Calculate(data4, 0, data4.Length);
var checksum5 = new Adler32RollingChecksumV3().Calculate(data5, 0, data5.Length);
// Assert
Assert.That(checksum1, Is.EqualTo(0x4ff907a1));
Assert.That(checksum2, Is.EqualTo(0x5206079b));
Assert.That(checksum3, Is.EqualTo(0x040f0fc1));
Assert.That(checksum4, Is.EqualTo(0x2d10357d));
Assert.That(checksum5, Is.EqualTo(0xa05ca509));
}
[Test]
public void Adler32RollingChecksumV3_RotatesChecksum()
{
//Arrange
const int len = 8 * 1024;
const int windowSize = len - 1;
var bytes = new byte[len];
new Random().NextBytes(bytes);
var adler = new Adler32RollingChecksumV3();
// Act
var rotatedChecksum = adler.Rotate(adler.Calculate(bytes, 0, windowSize), bytes[0], bytes[^1], windowSize);
var checksum = adler.Calculate(bytes, 1, windowSize);
// Assert
Assert.That(rotatedChecksum, Is.EqualTo(checksum));
}
}
================================================
FILE: source/FastRsync.Tests/BackwardCompatibilityTests.cs
================================================
using NUnit.Framework;
using System;
using System.IO;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
using FastRsync.Signature;
using NSubstitute;
using NUnit.Framework.Legacy;
namespace FastRsync.Tests;
[TestFixture]
public class BackwardCompatibilityTests
{
[Test]
[TestCase(".\\TestData\\patch112.bin")]
[TestCase(".\\TestData\\patch200.bin")]
[TestCase(".\\TestData\\patch231.bin")]
public async Task ApplyOldPatch(string deltaFileName)
{
// Arrange
const string baseFileName = ".\\TestData\\basefile.bin";
const string baseFile2Name = ".\\TestData\\basefile2.bin";
await using var baseFileStream = new FileStream(baseFileName, FileMode.Open);
await using var deltaFileStream = new FileStream(deltaFileName, FileMode.Open);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
// Act
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseFileStream, new BinaryDeltaReader(deltaFileStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
var newFileExpectedData = await File.ReadAllBytesAsync(baseFile2Name);
CollectionAssert.AreEqual(newFileExpectedData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
[Test]
[TestCase(".\\TestData\\signature112.bin")]
[TestCase(".\\TestData\\signature200.bin")]
[TestCase(".\\TestData\\signature231.bin")]
public async Task BuildPatchFromOldSignature(string signatureFileName)
{
// Arrange
const string baseFileName = ".\\TestData\\basefile.bin";
const string baseFile2Name = ".\\TestData\\basefile2.bin";
await using var signatureFileStream = new FileStream(signatureFileName, FileMode.Open);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
var patchedDataStream = new MemoryStream();
// Act
await using (var baseFile2Stream = new FileStream(baseFile2Name, FileMode.Open))
{
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(baseFile2Stream, new SignatureReader(signatureFileStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
await using var baseFileStream = new FileStream(baseFileName, FileMode.Open);
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseFileStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
}
// Assert
var newFileExpectedData = await File.ReadAllBytesAsync(baseFile2Name);
CollectionAssert.AreEqual(newFileExpectedData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
}
================================================
FILE: source/FastRsync.Tests/CommonAsserts.cs
================================================
using System.IO;
using FastRsync.Hash;
using FastRsync.Signature;
using NUnit.Framework;
namespace FastRsync.Tests;
class CommonAsserts
{
public static void ValidateSignature(Stream signatureStream, IHashAlgorithm hashAlgorithm, string baseFileHash, IRollingChecksum rollingAlgorithm)
{
signatureStream.Seek(0, SeekOrigin.Begin);
var sig = new SignatureReader(signatureStream, null).ReadSignature();
Assert.That(sig.Type, Is.EqualTo(RsyncFormatType.FastRsync));
Assert.That(sig.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name));
Assert.That(sig.Metadata.ChunkHashAlgorithm, Is.EqualTo(hashAlgorithm.Name));
Assert.That(sig.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes));
Assert.That(sig.RollingChecksumAlgorithm.Name, Is.EqualTo(rollingAlgorithm.Name));
Assert.That(sig.Metadata.RollingChecksumAlgorithm, Is.EqualTo(rollingAlgorithm.Name));
Assert.That(sig.Metadata.BaseFileHashAlgorithm, Is.EqualTo("MD5"));
Assert.That(sig.Metadata.BaseFileHash, Is.EqualTo(baseFileHash));
}
}
================================================
FILE: source/FastRsync.Tests/DeltaReaderTests.cs
================================================
using System.IO;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Signature;
using FastRsync.Tests.FastRsyncLegacy;
using NUnit.Framework;
namespace FastRsync.Tests;
[TestFixture]
public class DeltaReaderTests
{
/// <summary>
/// Metadata without BaseFileHashAlgorithm and BaseFileHash fields
/// </summary>
private static readonly byte[] FastRsyncLegacyMetadataDelta =
{
0x46, 0x52, 0x53, 0x4e, 0x43, 0x44, 0x4c, 0x54, 0x41, 0x01, 0x69, 0x7b, 0x22, 0x68, 0x61, 0x73,
0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a, 0x22, 0x58, 0x58, 0x48,
0x36, 0x34, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46, 0x69, 0x6c,
0x65, 0x48, 0x61, 0x73, 0x68, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x22, 0x3a,
0x22, 0x4d, 0x44, 0x35, 0x22, 0x2c, 0x22, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x46,
0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x3a, 0x22, 0x4e, 0x33, 0x48, 0x65, 0x65, 0x51,
0x62, 0x48, 0x52, 0x5a, 0x62, 0x65, 0x53, 0x47, 0x35, 0x4c, 0x4c, 0x50, 0x39, 0x46, 0x2f, 0x41,
0x3d, 0x3d, 0x22, 0x7d, 0x80, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x65, 0x63,
0x64
};
private static readonly string FastRsyncLegacyDeltaExpectedFileHash = "N3HeeQbHRZbeSG5LLP9F/A==";
[Test]
public void BinaryDeltaReader_ReadsLegacyDelta()
{
// Arrange
var deltaStream = new MemoryStream(FastRsyncLegacyMetadataDelta);
// Act
IDeltaReader target = new BinaryDeltaReader(deltaStream, null);
// Assert
var hashAlgorithm = SupportedAlgorithms.Hashing.Default();
Assert.That(target.Type, Is.EqualTo(RsyncFormatType.FastRsync));
Assert.That(target.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name));
Assert.That(target.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes));
Assert.That(target.Metadata.ExpectedFileHash, Is.EqualTo(FastRsyncLegacyDeltaExpectedFileHash));
Assert.That(target.Metadata.ExpectedFileHashAlgorithm, Is.EqualTo("MD5"));
Assert.That(target.Metadata.HashAlgorithm, Is.EqualTo(hashAlgorithm.Name));
Assert.That(target.Metadata.BaseFileHash, Is.Null);
Assert.That(target.Metadata.BaseFileHashAlgorithm, Is.Null);
}
[Test]
public void LegacyBinaryDeltaReader_ReadsDelta()
{
// Arrange
var (_, baseSignatureStream, _, newDataStream) = Utils.PrepareTestData(16974, 8452, SignatureBuilder.DefaultChunkSize);
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
deltaBuilder.BuildDelta(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
deltaStream.Seek(0, SeekOrigin.Begin);
// Act
var target = new BinaryDeltaReaderLegacy(deltaStream, null);
// Assert
var hashAlgorithm = SupportedAlgorithms.Hashing.Default();
Assert.That(target.HashAlgorithm.Name, Is.EqualTo(hashAlgorithm.Name));
Assert.That(target.HashAlgorithm.HashLengthInBytes, Is.EqualTo(hashAlgorithm.HashLengthInBytes));
Assert.That(target.Type, Is.EqualTo(RsyncFormatType.FastRsync));
Assert.That(target.Metadata.ExpectedFileHash, Is.Not.Null.And.Not.Empty);
Assert.That(target.Metadata.ExpectedFileHashAlgorithm, Is.EqualTo("MD5"));
Assert.That(target.Metadata.HashAlgorithm, Is.EqualTo(hashAlgorithm.Name));
}
}
================================================
FILE: source/FastRsync.Tests/FastRsync.Tests.csproj
================================================
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<None Remove="TestData\basefile.bin" />
<None Remove="TestData\basefile2.bin" />
<None Remove="TestData\patch112.bin" />
<None Remove="TestData\patch200.bin" />
<None Remove="TestData\patch231.bin" />
<None Remove="TestData\signature112.bin" />
<None Remove="TestData\signature200.bin" />
<None Remove="TestData\signature231.bin" />
</ItemGroup>
<ItemGroup>
<Content Include="TestData\basefile.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\basefile2.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\patch112.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\patch200.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\patch231.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\signature112.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\signature200.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="TestData\signature231.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FastRsync.Compression\FastRsync.Compression.csproj" />
<ProjectReference Include="..\FastRsync\FastRsync.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="NUnit" Version="4.4.0" />
<PackageReference Include="NUnit3TestAdapter" Version="6.1.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.27.0" />
<PackageReference Include="System.IO.Hashing" Version="10.0.1" />
<PackageReference Include="System.Data.HashFunction.Core" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.Interfaces" Version="2.0.0" />
<PackageReference Include="System.Data.HashFunction.xxHash" Version="2.0.0" />
</ItemGroup>
</Project>
================================================
FILE: source/FastRsync.Tests/FastRsyncLegacy/BinaryDeltaReaderLegacy.cs
================================================
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
using FastRsync.Hash;
using FastRsync.Signature;
using Newtonsoft.Json;
namespace FastRsync.Tests.FastRsyncLegacy;
internal class BinaryFormat
{
public const int SignatureFormatHeaderLength = 7; // OCTOSIG or FRSNCSG
public const int DeltaFormatHeaderLength = 9; // OCTODELTA or FRSNCDLTA
public const byte CopyCommand = 0x60;
public const byte DataCommand = 0x80;
}
internal class OctoBinaryFormat
{
public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG");
public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA");
public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>");
public const byte Version = 0x01;
}
internal class FastRsyncBinaryFormat
{
public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("FRSNCSG");
public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("FRSNCDLTA");
public const byte Version = 0x01;
}
public sealed class ProgressReport
{
public ProgressOperationType Operation { get; internal set; }
public long CurrentPosition { get; internal set; }
public long Total { get; internal set; }
}
internal class BinaryDeltaReaderLegacy : IDeltaReaderLegacy
{
private readonly BinaryReader reader;
private readonly IProgress<ProgressReport> progressReport;
private byte[] expectedHash;
private IHashAlgorithm hashAlgorithm;
private readonly int readBufferSize;
public BinaryDeltaReaderLegacy(Stream stream, IProgress<ProgressReport> progressHandler, int readBufferSize = 4 * 1024 * 1024)
{
this.reader = new BinaryReader(stream);
this.progressReport = progressHandler;
this.readBufferSize = readBufferSize;
}
private DeltaMetadataLegacy _metadata;
public DeltaMetadataLegacy Metadata
{
get
{
ReadMetadata();
return _metadata;
}
}
public RsyncFormatType Type { get; private set; }
public byte[] ExpectedHash
{
get
{
ReadMetadata();
return expectedHash;
}
}
public IHashAlgorithm HashAlgorithm
{
get
{
ReadMetadata();
return hashAlgorithm;
}
}
private void ReadMetadata()
{
if (_metadata != null)
return;
reader.BaseStream.Seek(0, SeekOrigin.Begin);
var header = reader.ReadBytes(BinaryFormat.DeltaFormatHeaderLength);
if (StructuralComparisons.StructuralEqualityComparer.Equals(FastRsyncBinaryFormat.DeltaHeader, header))
{
ReadFastRsyncDeltaHeader();
return;
}
if (StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.DeltaHeader, header))
{
ReadOctoDeltaHeader();
return;
}
throw new InvalidDataException("The delta file uses a different file format than this program can handle.");
}
private void ReadFastRsyncDeltaHeader()
{
var version = reader.ReadByte();
if (version != FastRsyncBinaryFormat.Version)
throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
var metadataStr = reader.ReadString();
_metadata = JsonConvert.DeserializeObject<DeltaMetadataLegacy>(metadataStr, NewtonsoftJsonSerializationSettings.JsonSettings);
hashAlgorithm = SupportedAlgorithms.Hashing.Create(_metadata.HashAlgorithm);
expectedHash = Convert.FromBase64String(_metadata.ExpectedFileHash);
Type = RsyncFormatType.FastRsync;
}
private void ReadOctoDeltaHeader()
{
var version = reader.ReadByte();
if (version != OctoBinaryFormat.Version)
throw new InvalidDataException("The delta file uses a newer file format than this program can handle.");
var hashAlgorithmName = reader.ReadString();
hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName);
var hashLength = reader.ReadInt32();
expectedHash = reader.ReadBytes(hashLength);
var endOfMeta = reader.ReadBytes(OctoBinaryFormat.EndOfMetadata.Length);
if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctoBinaryFormat.EndOfMetadata, endOfMeta))
throw new InvalidDataException("The delta file appears to be corrupt.");
_metadata = new DeltaMetadataLegacy
{
HashAlgorithm = hashAlgorithmName,
ExpectedFileHashAlgorithm = hashAlgorithmName,
ExpectedFileHash = Convert.ToBase64String(expectedHash)
};
Type = RsyncFormatType.Octodiff;
}
public void Apply(
Action<byte[]> writeData,
Action<long, long> copy)
{
var fileLength = reader.BaseStream.Length;
ReadMetadata();
while (reader.BaseStream.Position != fileLength)
{
var b = reader.ReadByte();
progressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.ApplyingDelta,
CurrentPosition = reader.BaseStream.Position,
Total = fileLength
});
if (b == BinaryFormat.CopyCommand)
{
var start = reader.ReadInt64();
var length = reader.ReadInt64();
copy(start, length);
}
else if (b == BinaryFormat.DataCommand)
{
var length = reader.ReadInt64();
long soFar = 0;
while (soFar < length)
{
var bytes = reader.ReadBytes((int)Math.Min(length - soFar, readBufferSize));
soFar += bytes.Length;
writeData(bytes);
}
}
}
}
public async Task ApplyAsync(
Func<byte[], Task> writeData,
Func<long, long, Task> copy)
{
var fileLength = reader.BaseStream.Length;
ReadMetadata();
var buffer = new byte[readBufferSize];
while (reader.BaseStream.Position != fileLength)
{
var b = reader.ReadByte();
progressReport?.Report(new ProgressReport
{
Operation = ProgressOperationType.ApplyingDelta,
CurrentPosition = reader.BaseStream.Position,
Total = fileLength
});
if (b == BinaryFormat.CopyCommand)
{
var start = reader.ReadInt64();
var length = reader.ReadInt64();
await copy(start, length).ConfigureAwait(false);
}
else if (b == BinaryFormat.DataCommand)
{
var length = reader.ReadInt64();
long soFar = 0;
while (soFar < length)
{
var bytesRead = await reader.BaseStream.ReadAsync(buffer, 0, (int)Math.Min(length - soFar, buffer.Length)).ConfigureAwait(false);
var bytes = buffer;
if (bytesRead != buffer.Length)
{
bytes = new byte[bytesRead];
Array.Copy(buffer, bytes, bytesRead);
}
soFar += bytes.Length;
await writeData(bytes).ConfigureAwait(false);
}
}
}
}
}
================================================
FILE: source/FastRsync.Tests/FastRsyncLegacy/DeltaMetadataLegacy.cs
================================================
namespace FastRsync.Tests.FastRsyncLegacy;
internal class DeltaMetadataLegacy
{
public string HashAlgorithm { get; set; }
public string ExpectedFileHashAlgorithm { get; set; }
public string ExpectedFileHash { get; set; }
}
================================================
FILE: source/FastRsync.Tests/FastRsyncLegacy/IDeltaReaderLegacy.cs
================================================
using System;
using System.Threading.Tasks;
using FastRsync.Hash;
using FastRsync.Signature;
namespace FastRsync.Tests.FastRsyncLegacy;
internal interface IDeltaReaderLegacy
{
byte[] ExpectedHash { get; }
IHashAlgorithm HashAlgorithm { get; }
DeltaMetadataLegacy Metadata { get; }
RsyncFormatType Type { get; }
void Apply(
Action<byte[]> writeData,
Action<long, long> copy
);
Task ApplyAsync(
Func<byte[], Task> writeData,
Func<long, long, Task> copy
);
}
================================================
FILE: source/FastRsync.Tests/FastRsyncLegacy/NewtonsoftJsonSerializationSettings.cs
================================================
using Newtonsoft.Json.Serialization;
using Newtonsoft.Json;
namespace FastRsync.Tests.FastRsyncLegacy;
public class NewtonsoftJsonSerializationSettings
{
static NewtonsoftJsonSerializationSettings()
{
JsonSettings = new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
Formatting = Formatting.None,
NullValueHandling = NullValueHandling.Ignore
};
}
public static JsonSerializerSettings JsonSettings { get; }
}
================================================
FILE: source/FastRsync.Tests/GZipTests.cs
================================================
using System;
using System.IO;
using System.IO.Compression;
using System.Threading;
using System.Threading.Tasks;
using FastRsync.Compression;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Signature;
using Newtonsoft.Json.Linq;
using NUnit.Framework;
namespace FastRsync.Tests;
[TestFixture]
public class GZipTests
{
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
public void GZip_CompressRandomData(int dataLength)
{
// Arrange
var data = new byte[dataLength];
// Main flow
CompressData(data);
}
[Test]
public void GZip_CompressData()
{
// Arrange
var data = File.ReadAllBytes(".\\TestData\\basefile.bin");
// Main flow
var destStream = CompressData(data);
// Additional asserts
Assert.That(destStream.Length, Is.EqualTo(6982738));
}
private static MemoryStream CompressData(byte[] data)
{
// Arrange
var srcStream = new MemoryStream(data);
var destStream = new MemoryStream();
// Act
GZip.Compress(srcStream, destStream);
// Assert
destStream.Seek(0, SeekOrigin.Begin);
var decompressedStream = new MemoryStream();
using (var gz = new GZipStream(destStream, CompressionMode.Decompress, true))
{
gz.CopyTo(decompressedStream);
}
var dataOutput = decompressedStream.ToArray();
Assert.That(dataOutput, Is.EqualTo(data));
return destStream;
}
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
public void GZip_CompressRandomData_RsyncSignatureAndPatch(int dataLength)
{
// Arrange
var dataBasis = new byte[dataLength];
new Random().NextBytes(dataBasis);
var basisStream = new MemoryStream(dataBasis);
var newFileStream = new MemoryStream();
newFileStream.Write(dataBasis, 10, dataLength * 4 / 5);
var newRandomData = new byte[dataLength * 2 / 5];
new Random().NextBytes(newRandomData);
newFileStream.Write(newRandomData, 0, newRandomData.Length);
newFileStream.Seek(0, SeekOrigin.Begin);
// Main flow
FullCompressedRsyncFlow(basisStream, newFileStream);
}
[Test]
public void GZip_CompressData_RsyncSignatureAndPatch()
{
// Arrange
// Arrange
var dataBasis = File.ReadAllBytes(".\\TestData\\basefile.bin");
var basisStream = new MemoryStream(dataBasis);
var newData = File.ReadAllBytes(".\\TestData\\basefile2.bin");
var newFileStream = new MemoryStream(newData);
// Main flow
var deltaStream = FullCompressedRsyncFlow(basisStream, newFileStream);
// Not calculated, taken from the valid flow
Assert.That(deltaStream.Length, Is.EqualTo(662628));
}
private static MemoryStream FullCompressedRsyncFlow(MemoryStream basisStream, MemoryStream newFileStream)
{
var basisStreamCompressed = new MemoryStream();
var basisStreamCompressedSignature = new MemoryStream();
var newFileStreamCompressed = new MemoryStream();
var deltaStream = new MemoryStream();
var patchedCompressedStream = new MemoryStream();
// Act
GZip.Compress(basisStream, basisStreamCompressed);
basisStreamCompressed.Seek(0, SeekOrigin.Begin);
var signatureBuilder = new SignatureBuilder();
signatureBuilder.Build(basisStreamCompressed, new SignatureWriter(basisStreamCompressedSignature));
basisStreamCompressedSignature.Seek(0, SeekOrigin.Begin);
GZip.Compress(newFileStream, newFileStreamCompressed);
newFileStreamCompressed.Seek(0, SeekOrigin.Begin);
var deltaBuilder = new DeltaBuilder();
deltaBuilder.BuildDelta(newFileStreamCompressed, new SignatureReader(basisStreamCompressedSignature, null),
new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
deltaStream.Seek(0, SeekOrigin.Begin);
var deltaApplier = new DeltaApplier
{
SkipHashCheck = true
};
var deltaReader = new BinaryDeltaReader(deltaStream, null);
deltaApplier.Apply(basisStreamCompressed, deltaReader, patchedCompressedStream);
deltaApplier.HashCheck(deltaReader, patchedCompressedStream);
// Assert
Assert.That(newFileStreamCompressed.ToArray(), Is.EqualTo(patchedCompressedStream.ToArray()));
patchedCompressedStream.Seek(0, SeekOrigin.Begin);
var decompressedStream = new MemoryStream();
using (var gz = new GZipStream(patchedCompressedStream, CompressionMode.Decompress))
{
gz.CopyTo(decompressedStream);
}
var dataOutput = decompressedStream.ToArray();
Assert.That(dataOutput, Is.EqualTo(newFileStream.ToArray()));
return deltaStream;
}
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
public async Task GZip_CompressRandomDataAsync_RsyncSignatureAndPatch(int dataLength)
{
// Arrange
var dataBasis = new byte[dataLength];
new Random().NextBytes(dataBasis);
var newFileStream = new MemoryStream();
newFileStream.Write(dataBasis, 10, dataLength * 4 / 5);
var newRandomData = new byte[dataLength * 2 / 5];
new Random().NextBytes(newRandomData);
newFileStream.Write(newRandomData, 0, newRandomData.Length);
newFileStream.Seek(0, SeekOrigin.Begin);
var basisStream = new MemoryStream(dataBasis);
// Main flow
await FullCompressedRsyncFlowAsync(basisStream, newFileStream);
}
[Test]
public async Task GZip_CompressDataAsync_RsyncSignatureAndPatch()
{
// Arrange
var dataBasis = File.ReadAllBytes(".\\TestData\\basefile.bin");
var basisStream = new MemoryStream(dataBasis);
var newData = File.ReadAllBytes(".\\TestData\\basefile2.bin");
var newFileStream = new MemoryStream(newData);
// Main flow
var deltaStream = await FullCompressedRsyncFlowAsync(basisStream, newFileStream);
// Not calculated, taken from the valid flow
Assert.That(deltaStream.Length, Is.EqualTo(662628));
}
private static async Task<MemoryStream> FullCompressedRsyncFlowAsync(MemoryStream basisStream, MemoryStream newFileStream)
{
// Arrange
var basisStreamCompressed = new MemoryStream();
var basisStreamCompressedSignature = new MemoryStream();
var newFileStreamCompressed = new MemoryStream();
var deltaStream = new MemoryStream();
var patchedCompressedStream = new MemoryStream();
// Act
await GZip.CompressAsync(basisStream, basisStreamCompressed);
basisStreamCompressed.Seek(0, SeekOrigin.Begin);
var signatureBuilder = new SignatureBuilder();
await signatureBuilder.BuildAsync(basisStreamCompressed, new SignatureWriter(basisStreamCompressedSignature));
basisStreamCompressedSignature.Seek(0, SeekOrigin.Begin);
await GZip.CompressAsync(newFileStream, newFileStreamCompressed);
newFileStreamCompressed.Seek(0, SeekOrigin.Begin);
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newFileStreamCompressed,
new SignatureReader(basisStreamCompressedSignature, null),
new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
deltaStream.Seek(0, SeekOrigin.Begin);
var deltaApplier = new DeltaApplier
{
SkipHashCheck = true
};
var deltaReader = new BinaryDeltaReader(deltaStream, null);
await deltaApplier.ApplyAsync(basisStreamCompressed, deltaReader, patchedCompressedStream);
await deltaApplier.HashCheckAsync(deltaReader, patchedCompressedStream);
// Assert
Assert.That(newFileStreamCompressed.ToArray(), Is.EqualTo(patchedCompressedStream.ToArray()));
patchedCompressedStream.Seek(0, SeekOrigin.Begin);
var decompressedStream = new MemoryStream();
await using (var gz = new GZipStream(patchedCompressedStream, CompressionMode.Decompress))
{
await gz.CopyToAsync(decompressedStream);
}
Assert.That(newFileStream.ToArray(), Is.EqualTo(decompressedStream.ToArray()));
return deltaStream;
}
}
================================================
FILE: source/FastRsync.Tests/HashTests.cs
================================================
using NUnit.Framework;
using System;
using System.IO;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Specialized;
using System.Data.HashFunction.xxHash;
using System.IO.Hashing;
namespace FastRsync.Tests;
[TestFixture]
public class HashTests
{
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
[TestCase(6 * 1024 * 1024)]
[TestCase(7 * 1023 * 1017)]
[TestCase(100 * 333 * 333)]
public void XxHash64StaticBackwardCompatibility(int dataLength)
{
// Arrange
var data = new byte[dataLength];
new Random().NextBytes(data);
// Act
var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig
{
HashSizeInBits = 64
});
var dataXxHash64Dest = dataXxHash.ComputeHash(data).Hash;
var systemXxHash64Dest = XxHash64.Hash(data);
Array.Reverse(systemXxHash64Dest);
// Assert
Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest));
}
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
[TestCase(6 * 1024 * 1024)]
[TestCase(7 * 1023 * 1017)]
[TestCase(100 * 333 * 333)]
public void XxHash64BackwardCompatibility(int dataLength)
{
// Arrange
var data = new byte[dataLength];
new Random().NextBytes(data);
// Act
var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig
{
HashSizeInBits = 64
});
NonCryptographicHashAlgorithm systemXxHash = new XxHash64();
var dataXxHash64Dest = dataXxHash.ComputeHash(data).Hash;
systemXxHash.Append(data);
var systemXxHash64Dest = systemXxHash.GetHashAndReset();
Array.Reverse(systemXxHash64Dest);
// Assert
Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest));
}
[Test]
[TestCase(120)]
[TestCase(5 * 1024 * 1024)]
[TestCase(100 * 333 * 333)]
public async Task XxHash64StreamBackwardCompatibility(int dataLength)
{
// Arrange
var data = new byte[dataLength];
new Random().NextBytes(data);
var streamdata = new MemoryStream(data);
const string azureBlobStorageConnectionString = "UseDevelopmentStorage=true;";
var azureBlobServiceClient = new BlobServiceClient(azureBlobStorageConnectionString);
var azureBlobContainerClient = azureBlobServiceClient.GetBlobContainerClient("hashbenchmark");
await azureBlobContainerClient.CreateIfNotExistsAsync();
var azureBlob = azureBlobContainerClient.GetBlockBlobClient("XxHash64StreamBackwardCompatibility_" + dataLength);
await azureBlob.UploadAsync(streamdata);
await using var blobstreamDataXxHash = await azureBlob.OpenReadAsync();
await using var blobstreamSystemXxHash = await azureBlob.OpenReadAsync();
// Act
var dataXxHash = xxHashFactory.Instance.Create(new xxHashConfig
{
HashSizeInBits = 64
});
NonCryptographicHashAlgorithm systemXxHash = new XxHash64();
var dataXxHash64Dest = (await dataXxHash.ComputeHashAsync(blobstreamDataXxHash)).Hash;
await systemXxHash.AppendAsync(blobstreamSystemXxHash);
var systemXxHash64Dest = systemXxHash.GetHashAndReset();
Array.Reverse(systemXxHash64Dest);
// Assert
Assert.That(systemXxHash64Dest, Is.EqualTo(dataXxHash64Dest));
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffDeltaWriter.cs
================================================
using System.IO;
using System.Threading.Tasks;
using FastRsync.Delta;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
public interface IOctodiffDeltaWriter
{
void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash);
void WriteCopyCommand(DataRange segment);
void WriteDataCommand(Stream source, long offset, long length);
void Finish();
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffSignatureWriter.cs
================================================
using System.IO;
using FastRsync.Core;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
public interface IOctodiffSignatureWriter
{
void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm);
void WriteChunk(ChunkSignature signature);
}
public class OctodiffSignatureWriter(Stream signatureStream) : IOctodiffSignatureWriter
{
private readonly BinaryWriter signaturebw = new(signatureStream);
public void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm)
{
signaturebw.Write(OctodiffBinaryFormat.SignatureHeader);
signaturebw.Write(OctodiffBinaryFormat.Version);
signaturebw.Write(hashAlgorithm.Name);
signaturebw.Write(rollingChecksumAlgorithm.Name);
signaturebw.Write(OctodiffBinaryFormat.EndOfMetadata);
}
public void WriteChunk(ChunkSignature signature)
{
signaturebw.Write(signature.Length);
signaturebw.Write(signature.RollingChecksum);
signaturebw.Write(signature.Hash);
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffAggregateCopyOperationsDecorator.cs
================================================
using System.IO;
using FastRsync.Delta;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
// This decorator turns any sequential copy operations into a single operation, reducing
// the size of the delta file.
// For example:
// Copy: 0x0000 - 0x0400
// Copy: 0x0401 - 0x0800
// Copy: 0x0801 - 0x0C00
// Gets turned into:
// Copy: 0x0000 - 0x0C00
public class OctodiffAggregateCopyOperationsDecorator : IOctodiffDeltaWriter
{
private readonly IOctodiffDeltaWriter decorated;
private DataRange bufferedCopy;
public OctodiffAggregateCopyOperationsDecorator(IOctodiffDeltaWriter decorated)
{
this.decorated = decorated;
}
public void WriteDataCommand(Stream source, long offset, long length)
{
FlushCurrentCopyCommand();
decorated.WriteDataCommand(source, offset, length);
}
public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash)
{
decorated.WriteMetadata(hashAlgorithm, expectedNewFileHash);
}
public void WriteCopyCommand(DataRange chunk)
{
if (bufferedCopy.Length > 0 && bufferedCopy.StartOffset + bufferedCopy.Length == chunk.StartOffset)
{
bufferedCopy.Length += chunk.Length;
}
else
{
FlushCurrentCopyCommand();
bufferedCopy = chunk;
}
}
private void FlushCurrentCopyCommand()
{
if (bufferedCopy.Length <= 0)
{
return;
}
decorated.WriteCopyCommand(bufferedCopy);
bufferedCopy = new DataRange();
}
public void Finish()
{
FlushCurrentCopyCommand();
decorated.Finish();
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryDeltaWriter.cs
================================================
using System;
using System.IO;
using FastRsync.Delta;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
public class OctodiffBinaryDeltaWriter : IOctodiffDeltaWriter
{
private readonly BinaryWriter writer;
private readonly int readWriteBufferSize;
public OctodiffBinaryDeltaWriter(Stream stream, int readWriteBufferSize = 1024 * 1024)
{
writer = new BinaryWriter(stream);
this.readWriteBufferSize = readWriteBufferSize;
}
public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash)
{
writer.Write(OctodiffBinaryFormat.DeltaHeader);
writer.Write(OctodiffBinaryFormat.Version);
writer.Write(hashAlgorithm.Name);
writer.Write(expectedNewFileHash.Length);
writer.Write(expectedNewFileHash);
writer.Write(OctodiffBinaryFormat.EndOfMetadata);
}
public void WriteCopyCommand(DataRange segment)
{
writer.Write(OctodiffBinaryFormat.CopyCommand);
writer.Write(segment.StartOffset);
writer.Write(segment.Length);
}
public void WriteDataCommand(Stream source, long offset, long length)
{
writer.Write(OctodiffBinaryFormat.DataCommand);
writer.Write(length);
var originalPosition = source.Position;
try
{
source.Seek(offset, SeekOrigin.Begin);
var buffer = new byte[Math.Min((int)length, readWriteBufferSize)];
int read;
long soFar = 0;
while ((read = source.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0)
{
soFar += read;
writer.Write(buffer, 0, read);
}
}
finally
{
source.Seek(originalPosition, SeekOrigin.Begin);
}
}
public void Finish()
{
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryFormat.cs
================================================
using System.Text;
namespace FastRsync.Tests.OctodiffLegacy;
class OctodiffBinaryFormat
{
public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG");
public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA");
public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>");
public const byte CopyCommand = 0x60;
public const byte DataCommand = 0x80;
public const byte Version = 0x01;
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffDeltaBuilder.cs
================================================
using System.Collections;
using System.Collections.Generic;
using System.IO;
using FastRsync.Core;
using FastRsync.Delta;
namespace FastRsync.Tests.OctodiffLegacy;
public class OctodiffDeltaBuilder
{
private readonly int readBufferSize;
public OctodiffDeltaBuilder(int readBufferSize = 4 * 1024 * 1024)
{
this.readBufferSize = readBufferSize;
}
public void BuildDelta(Stream newFileStream, IOctodiffSignatureReader signatureReader, IOctodiffDeltaWriter deltaWriter)
{
var signature = signatureReader.ReadSignature();
var chunks = signature.Chunks;
var newFileHash = signature.HashAlgorithm.ComputeHash(newFileStream);
newFileStream.Seek(0, SeekOrigin.Begin);
deltaWriter.WriteMetadata(signature.HashAlgorithm, newFileHash);
chunks = OrderChunksByChecksum(chunks);
var chunkMap = CreateChunkMap(chunks, out int maxChunkSize, out int minChunkSize);
var buffer = new byte[readBufferSize];
long lastMatchPosition = 0;
while (true)
{
var startPosition = newFileStream.Position;
var read = newFileStream.Read(buffer, 0, buffer.Length);
if (read <= 0)
break;
var checksumAlgorithm = signature.RollingChecksumAlgorithm;
uint checksum = 0;
var remainingPossibleChunkSize = maxChunkSize;
for (var i = 0; i < read - minChunkSize + 1; i++)
{
var readSoFar = startPosition + i;
var remainingBytes = read - i;
if (remainingBytes < maxChunkSize)
{
remainingPossibleChunkSize = minChunkSize;
}
if (i == 0 || remainingBytes < maxChunkSize)
{
checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize);
}
else
{
var remove = buffer[i - 1];
var add = buffer[i + remainingPossibleChunkSize - 1];
checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize);
}
if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize)
continue;
if (!chunkMap.ContainsKey(checksum))
continue;
var startIndex = chunkMap[checksum];
for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++)
{
var chunk = chunks[j];
var hash = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize);
if (StructuralComparisons.StructuralEqualityComparer.Equals(hash, chunks[j].Hash))
{
readSoFar = readSoFar + remainingPossibleChunkSize;
var missing = readSoFar - lastMatchPosition;
if (missing > remainingPossibleChunkSize)
{
deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize);
}
deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length));
lastMatchPosition = readSoFar;
break;
}
}
}
if (read < buffer.Length)
{
break;
}
newFileStream.Position = newFileStream.Position - maxChunkSize + 1;
}
if (newFileStream.Length != lastMatchPosition)
{
deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition);
}
deltaWriter.Finish();
}
private class ChunkSignatureChecksumComparer : IComparer<ChunkSignature>
{
public int Compare(ChunkSignature x, ChunkSignature y)
{
var comparison = x.RollingChecksum.CompareTo(y.RollingChecksum);
return comparison == 0 ? x.StartOffset.CompareTo(y.StartOffset) : comparison;
}
}
private static List<ChunkSignature> OrderChunksByChecksum(List<ChunkSignature> chunks)
{
chunks.Sort(new ChunkSignatureChecksumComparer());
return chunks;
}
private Dictionary<uint, int> CreateChunkMap(IList<ChunkSignature> chunks, out int maxChunkSize, out int minChunkSize)
{
maxChunkSize = 0;
minChunkSize = int.MaxValue;
var chunkMap = new Dictionary<uint, int>();
for (var i = 0; i < chunks.Count; i++)
{
var chunk = chunks[i];
if (chunk.Length > maxChunkSize)
{
maxChunkSize = chunk.Length;
}
if (chunk.Length < minChunkSize)
{
minChunkSize = chunk.Length;
}
if (!chunkMap.ContainsKey(chunk.RollingChecksum))
{
chunkMap[chunk.RollingChecksum] = i;
}
}
return chunkMap;
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignature.cs
================================================
using System.Collections.Generic;
using FastRsync.Core;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
public class OctodiffSignature
{
public OctodiffSignature(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm)
{
HashAlgorithm = hashAlgorithm;
RollingChecksumAlgorithm = rollingChecksumAlgorithm;
Chunks = new List<ChunkSignature>();
}
public IHashAlgorithm HashAlgorithm { get; private set; }
public IRollingChecksum RollingChecksumAlgorithm { get; private set; }
public List<ChunkSignature> Chunks { get; private set; }
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureBuilder.cs
================================================
using System;
using System.IO;
using FastRsync.Core;
using FastRsync.Hash;
namespace FastRsync.Tests.OctodiffLegacy;
public class OctodiffSignatureBuilder
{
public const short MinimumChunkSize = 128;
public const short DefaultChunkSize = 2048;
public const short MaximumChunkSize = 31 * 1024;
private short chunkSize;
public OctodiffSignatureBuilder() : this(SupportedAlgorithms.Hashing.Sha1(), SupportedAlgorithms.Checksum.Default())
{
}
public OctodiffSignatureBuilder(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm)
{
HashAlgorithm = hashAlgorithm;
RollingChecksumAlgorithm = rollingChecksumAlgorithm;
ChunkSize = DefaultChunkSize;
}
public IHashAlgorithm HashAlgorithm { get; set; }
public IRollingChecksum RollingChecksumAlgorithm { get; set; }
public short ChunkSize
{
get => chunkSize;
set
{
if (value < MinimumChunkSize)
throw new ArgumentException($"Chunk size cannot be less than {MinimumChunkSize}");
if (value > MaximumChunkSize)
throw new ArgumentException($"Chunk size cannot be exceed {MaximumChunkSize}");
chunkSize = value;
}
}
public void Build(Stream stream, IOctodiffSignatureWriter signatureWriter)
{
WriteMetadata(stream, signatureWriter);
WriteChunkSignatures(stream, signatureWriter);
}
private void WriteMetadata(Stream stream, IOctodiffSignatureWriter signatureWriter)
{
stream.Seek(0, SeekOrigin.Begin);
signatureWriter.WriteMetadata(HashAlgorithm, RollingChecksumAlgorithm);
}
private void WriteChunkSignatures(Stream stream, IOctodiffSignatureWriter signatureWriter)
{
var checksumAlgorithm = RollingChecksumAlgorithm;
var hashAlgorithm = HashAlgorithm;
stream.Seek(0, SeekOrigin.Begin);
long start = 0;
int read;
var block = new byte[ChunkSize];
while ((read = stream.Read(block, 0, block.Length)) > 0)
{
signatureWriter.WriteChunk(new ChunkSignature
{
StartOffset = start,
Length = (short)read,
Hash = hashAlgorithm.ComputeHash(block, 0, read),
RollingChecksum = checksumAlgorithm.Calculate(block, 0, read)
});
start += read;
}
}
}
================================================
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureReader.cs
================================================
using System;
using System.Collections;
using System.IO;
using FastRsync.Core;
using FastRsync.Diagnostics;
namespace FastRsync.Tests.OctodiffLegacy;
public interface IOctodiffSignatureReader
{
OctodiffSignature ReadSignature();
}
public class OctodiffSignatureReader : IOctodiffSignatureReader
{
private readonly BinaryReader reader;
public OctodiffSignatureReader(Stream stream, IProgress<ProgressReport> progressHandler)
{
this.reader = new BinaryReader(stream);
}
public OctodiffSignature ReadSignature()
{
var header = reader.ReadBytes(OctodiffBinaryFormat.SignatureHeader.Length);
if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctodiffBinaryFormat.SignatureHeader, header))
throw new InvalidDataException("The signature file appears to be corrupt.");
var version = reader.ReadByte();
if (version != OctodiffBinaryFormat.Version)
throw new InvalidDataException("The signature file uses a newer file format than this program can handle.");
var hashAlgorithm = reader.ReadString();
var rollingChecksumAlgorithm = reader.ReadString();
var endOfMeta = reader.ReadBytes(OctodiffBinaryFormat.EndOfMetadata.Length);
if (!StructuralComparisons.StructuralEqualityComparer.Equals(OctodiffBinaryFormat.EndOfMetadata, endOfMeta))
throw new InvalidDataException("The signature file appears to be corrupt.");
var hashAlgo = SupportedAlgorithms.Hashing.Create(hashAlgorithm);
var signature = new OctodiffSignature(
hashAlgo,
SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm));
var expectedHashLength = hashAlgo.HashLengthInBytes;
long start = 0;
var fileLength = reader.BaseStream.Length;
var remainingBytes = fileLength - reader.BaseStream.Position;
var signatureSize = sizeof(ushort) + sizeof(uint) + expectedHashLength;
if (remainingBytes % signatureSize != 0)
throw new InvalidDataException("The signature file appears to be corrupt; at least one chunk has data missing.");
while (reader.BaseStream.Position < fileLength - 1)
{
var length = reader.ReadInt16();
var checksum = reader.ReadUInt32();
var chunkHash = reader.ReadBytes(expectedHashLength);
signature.Chunks.Add(new ChunkSignature
{
StartOffset = start,
Length = length,
RollingChecksum = checksum,
Hash = chunkHash
});
start += length;
}
return signature;
}
}
================================================
FILE: source/FastRsync.Tests/PatchingAsyncTests.cs
================================================
using System;
using System.IO;
using System.Threading.Tasks;
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
using FastRsync.Hash;
using FastRsync.Signature;
using FastRsync.Tests.OctodiffLegacy;
using NSubstitute;
using NUnit.Framework;
using NUnit.Framework.Legacy;
namespace FastRsync.Tests;
[TestFixture]
public class PatchingAsyncTests
{
[Test]
[TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")]
[TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH3", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH3", "Adler32")]
[TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH3", "Adler32")]
[TestCase(1378, 129, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")]
[TestCase(1378, 129, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")]
[TestCase(1378, 129, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")]
[TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")]
[TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")]
[TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")]
[TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize, "XXH64", "Adler32V2")]
[TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize, "XXH64", "Adler32V2")]
[TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize, "XXH64", "Adler32V2")]
public async Task PatchingAsyncXXHash_ForNewData_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize, string signatureHashingAlgorithm, string rollingChecksumAlgorithm)
{
// Arrange
var signatureHash = SupportedAlgorithms.Hashing.Create(signatureHashingAlgorithm);
var rollingChecksum = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm);
var (baseDataStream, baseSignatureStream, newData, newDataStream) = await PrepareTestDataAsync(baseNumberOfBytes, newDataNumberOfBytes, chunkSize, rollingChecksum, signatureHash).ConfigureAwait(false);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
// Act
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
[Test]
[TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)]
[TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)]
[TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)]
public async Task PatchingAsyncXXHash_ForOctodiffSignature_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize)
{
// Arrange
var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
// Act
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
[Test]
[TestCase(1378, 129, SignatureBuilder.MinimumChunkSize)]
[TestCase(1378, 129, SignatureBuilder.DefaultChunkSize)]
[TestCase(1378, 129, SignatureBuilder.MaximumChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.MinimumChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.DefaultChunkSize)]
[TestCase(16974, 8452, SignatureBuilder.MaximumChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.MinimumChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.DefaultChunkSize)]
[TestCase(6666, 6666, SignatureBuilder.MaximumChunkSize)]
public async Task PatchingAsyncXXHash_ForOctodiffSignatureAndPatch_PatchesFile(int baseNumberOfBytes, int newDataNumberOfBytes, short chunkSize)
{
// Arrange
var (baseDataStream, baseSignatureStream, newData, newDataStream) = PrepareTestDataWithOctodiffSignature(baseNumberOfBytes, newDataNumberOfBytes, chunkSize);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
var deltaStream = new MemoryStream();
var deltaBuilder = new OctodiffDeltaBuilder();
deltaBuilder.BuildDelta(newDataStream, new OctodiffSignatureReader(baseSignatureStream, null), new OctodiffAggregateCopyOperationsDecorator(new OctodiffBinaryDeltaWriter(deltaStream)));
deltaStream.Seek(0, SeekOrigin.Begin);
// Act
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(newData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
[Test]
[TestCase(8467, SignatureBuilder.DefaultChunkSize)]
public async Task PatchingAsyncXXHash_ForTheSameData_PatchesFile(int numberOfBytes, short chunkSize)
{
// Arrange
var baseData = new byte[numberOfBytes];
new Random().NextBytes(baseData);
var baseDataStream = new MemoryStream(baseData);
var baseSignatureStream = new MemoryStream();
var signatureBuilder = new SignatureBuilder
{
ChunkSize = chunkSize
};
signatureBuilder.Build(baseDataStream, new SignatureWriter(baseSignatureStream));
baseSignatureStream.Seek(0, SeekOrigin.Begin);
var newDataStream = new MemoryStream(baseData);
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
// Act
var deltaStream = new MemoryStream();
var deltaBuilder = new DeltaBuilder();
await deltaBuilder.BuildDeltaAsync(newDataStream, new SignatureReader(baseSignatureStream, null), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))).ConfigureAwait(false);
deltaStream.Seek(0, SeekOrigin.Begin);
var patchedDataStream = new MemoryStream();
var deltaApplier = new DeltaApplier();
await deltaApplier.ApplyAsync(baseDataStream, new BinaryDeltaReader(deltaStream, progressReporter), patchedDataStream).ConfigureAwait(false);
// Assert
CollectionAssert.AreEqual(baseData, patchedDataStream.ToArray());
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
private static async Task<(MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream)> PrepareTestDataAsync(int baseNumberOfBytes, int newDataNumberOfBytes,
short chunkSize, Hash.IRollingChecksum rollingChecksumAlg, IHashAlgorithm signatureHashingAlg)
{
var baseData = new byte[baseNumberOfBytes];
new Random().NextBytes(baseData);
var baseDataStream = new MemoryStream(baseData);
var baseSignatureStream = new MemoryStream();
var signatureBuilder = new SignatureBuilder(signatureHashingAlg, rollingChecksumAlg)
{
ChunkSize = chunkSize
};
await signatureBuilder.BuildAsync(baseDataStream, new SignatureWriter(baseSignatureStream)).ConfigureAwait(false);
baseSignatureStream.Seek(0, SeekOrigin.Begin);
var newData = new byte[newDataNumberOfBytes];
new Random().NextBytes(newData);
var newDataStream = new MemoryStream(newData);
return (baseDataStream, baseSignatureStream, newData, newDataStream);
}
private static (MemoryStream baseDataStream, MemoryStream baseSignatureStream, byte[] newData, MemoryStream newDataStream) PrepareTestDataWithOctodiffSignature(int baseNumberOfBytes, int newDataNumberOfBytes,
short chunkSize)
{
var baseData = new byte[baseNumberOfBytes];
new Random().NextBytes(baseData);
var baseDataStream = new MemoryStream(baseData);
var baseSignatureStream = new MemoryStream();
var signatureBuilder = new OctodiffSignatureBuilder
{
ChunkSize = chunkSize
};
signatureBuilder.Build(baseDataStream, new OctodiffSignatureWriter(baseSignatureStream));
baseSignatureStream.Seek(0, SeekOrigin.Begin);
var newData = new byte[newDataNumberOfBytes];
new Random().NextBytes(newData);
var newDataStream = new MemoryStream(newData);
return (baseDataStream, baseSignatureStream, newData, newDataStream);
}
}
================================================
FILE: source/FastRsync.Tests/PatchingBigFilesTests.cs
================================================
using FastRsync.Core;
using FastRsync.Delta;
using FastRsync.Diagnostics;
using FastRsync.Signature;
using NSubstitute;
using NUnit.Framework;
using System;
using System.IO;
using System.Security.Cryptography;
using FastRsync.Hash;
namespace FastRsync.Tests;
[TestFixture]
public class PatchingBigFilesTests
{
[Test]
[TestCase(2050, 2790, "XXH64", "Adler32")]
[TestCase(2050, 2790, "XXH3", "Adler32")]
[TestCase(2050, 2790, "XXH64", "Adler32V3")]
[TestCase(2050, 2790, "XXH3", "Adler32V3")]
public void PatchingSyncXXHash_BigFile(int baseNumberOfMBytes, int newNumberOfMBytes, string signatureHashingAlgorithm, string rollingChecksumAlgorithm)
{
// Arrange
var signatureHash = SupportedAlgorithms.Hashing.Create(signatureHashingAlgorithm);
var rollingChecksum = SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm);
var baseFileName = Path.GetTempFileName();
var newFileName = Path.GetTempFileName();
try
{
var deltaFileName = Path.GetTempFileName();
var patchedFileName = Path.GetTempFileName();
var progressReporter = Substitute.For<IProgress<ProgressReport>>();
{
var buffer = new byte[16384];
var rng = new Random();
using var baseFileStream = new FileStream(baseFileName, FileMode.Create);
for (var i = 0; i < baseNumberOfMBytes * 64; i++)
{
rng.NextBytes(buffer);
baseFileStream.Write(buffer, 0, buffer.Length);
}
baseFileStream.Seek(0, SeekOrigin.Begin);
using var newFileStream = new FileStream(newFileName, FileMode.Create);
for (var i = 0; i < baseNumberOfMBytes * 64; i++)
{
rng.NextBytes(buffer);
newFileStream.Write(buffer, 0, buffer.Length);
}
newFileStream.Seek(0, SeekOrigin.Begin);
using var baseSignatureStream = PrepareTestData(baseFileStream, signatureHash, rollingChecksum);
// Act
using var deltaStream = new FileStream(deltaFileName, FileMode.OpenOrCreate);
using var patchedDataStream = new FileStream(patchedFileName, FileMode.OpenOrCreate);
var deltaBuilder = new DeltaBuilder();
deltaBuilder.BuildDelta(newFileStream, new SignatureReader(baseSignatureStream, null),
new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream)));
deltaStream.Seek(0, SeekOrigin.Begin);
var deltaApplier = new DeltaApplier();
deltaApplier.Apply(baseFileStream, new BinaryDeltaReader(deltaStream, progressReporter),
patchedDataStream);
}
// Assert
Assert.That(new FileInfo(newFileName).Length, Is.EqualTo(new FileInfo(patchedFileName).Length));
Assert.That(CompareFilesByHash(newFileName, patchedFileName), Is.True);
progressReporter.Received().Report(Arg.Any<ProgressReport>());
}
catch (Exception)
{
Assert.Fail();
}
finally
{
File.Delete(baseFileName);
File.Delete(newFileName);
}
}
public static Stream PrepareTestData(Stream baseDataStream, IHashAlgorithm signatureHashingAlgorithm, IRollingChecksum rollingChecksumAlgorithm)
{
var baseSignatureStream = new MemoryStream();
var signatureBuilder = new Sig
gitextract_bd403tkm/
├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
└── source/
├── .editorconfig
├── FastRsync/
│ ├── Core/
│ │ ├── AggregateCopyOperationsDecorator.cs
│ │ ├── BinaryFormat.cs
│ │ ├── ChunkSignature.cs
│ │ ├── ChunkSignatureChecksumComparer.cs
│ │ ├── JsonContext.cs
│ │ ├── JsonSerializationSettings.cs
│ │ └── SupportedAlgorithms.cs
│ ├── Delta/
│ │ ├── BinaryDeltaReader.cs
│ │ ├── BinaryDeltaWriter.cs
│ │ ├── DataRange.cs
│ │ ├── DeltaApplier.cs
│ │ ├── DeltaBuilder.cs
│ │ ├── DeltaMetadata.cs
│ │ ├── IDeltaReader.cs
│ │ └── IDeltaWriter.cs
│ ├── Diagnostics/
│ │ ├── ConsoleProgressReporter.cs
│ │ └── ProgressReport.cs
│ ├── FastRsync.csproj
│ ├── Hash/
│ │ ├── Adler32RollingChecksum.cs
│ │ ├── Adler32RollingChecksumV2.cs
│ │ ├── Adler32RollingChecksumV3.cs
│ │ ├── CryptographyHashAlgorithmWrapper.cs
│ │ ├── IHashAlgorithm.cs
│ │ ├── IRollingChecksum.cs
│ │ └── NonCryptographicHashAlgorithmWrapper.cs
│ └── Signature/
│ ├── ISignatureReader.cs
│ ├── ISignatureWriter.cs
│ ├── Signature.cs
│ ├── SignatureBuilder.cs
│ ├── SignatureReader.cs
│ └── SignatureWriter.cs
├── FastRsync.BackwardCompatibilityTests/
│ ├── BackwardCompatibilityTests.cs
│ └── FastRsync.BackwardCompatibilityTests.csproj
├── FastRsync.Benchmarks/
│ ├── BuildPatchBenchmark.cs
│ ├── FastRsync.Benchmarks.csproj
│ ├── HashBenchmark.cs
│ ├── Program.cs
│ ├── RollingChecksumBenchmark.cs
│ └── SignatureBenchmark.cs
├── FastRsync.Compression/
│ ├── FastRsync.Compression.csproj
│ └── GZip.cs
├── FastRsync.Tests/
│ ├── Adler32RollingChecksumAlgorithmsTests.cs
│ ├── Adler32RollingChecksumTests.cs
│ ├── Adler32RollingChecksumV2Tests.cs
│ ├── Adler32RollingChecksumV3Tests.cs
│ ├── BackwardCompatibilityTests.cs
│ ├── CommonAsserts.cs
│ ├── DeltaReaderTests.cs
│ ├── FastRsync.Tests.csproj
│ ├── FastRsyncLegacy/
│ │ ├── BinaryDeltaReaderLegacy.cs
│ │ ├── DeltaMetadataLegacy.cs
│ │ ├── IDeltaReaderLegacy.cs
│ │ └── NewtonsoftJsonSerializationSettings.cs
│ ├── GZipTests.cs
│ ├── HashTests.cs
│ ├── OctodiffLegacy/
│ │ ├── IOctodiffDeltaWriter.cs
│ │ ├── IOctodiffSignatureWriter.cs
│ │ ├── OctodiffAggregateCopyOperationsDecorator.cs
│ │ ├── OctodiffBinaryDeltaWriter.cs
│ │ ├── OctodiffBinaryFormat.cs
│ │ ├── OctodiffDeltaBuilder.cs
│ │ ├── OctodiffSignature.cs
│ │ ├── OctodiffSignatureBuilder.cs
│ │ └── OctodiffSignatureReader.cs
│ ├── PatchingAsyncTests.cs
│ ├── PatchingBigFilesTests.cs
│ ├── PatchingSyncTests.cs
│ ├── SignatureBuilderAsyncRandomDataTests.cs
│ ├── SignatureBuilderSyncRandomDataTests.cs
│ ├── SignatureBuilderTests.cs
│ ├── SignatureReaderTests.cs
│ └── Utils.cs
├── FastRsync.sln
├── Octodiff/
│ ├── CommandLine/
│ │ ├── DeltaCommand.cs
│ │ ├── ExplainDeltaCommand.cs
│ │ ├── HelpCommand.cs
│ │ ├── PatchCommand.cs
│ │ ├── SignatureCommand.cs
│ │ └── Support/
│ │ ├── CommandAttribute.cs
│ │ ├── CommandException.cs
│ │ ├── CommandLocator.cs
│ │ ├── ICommand.cs
│ │ ├── ICommandLocator.cs
│ │ ├── ICommandMetadata.cs
│ │ └── NDesk.Options.cs
│ ├── Octodiff.csproj
│ ├── OctodiffProgram.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ └── app.config
├── Octodiff.Tests/
│ ├── DeltaFixture.cs
│ ├── HelpFixture.cs
│ ├── Octodiff.Tests.csproj
│ ├── PackageGenerator.cs
│ ├── PatchFixture.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ ├── SignatureFixture.cs
│ ├── Timings.cs
│ └── Util/
│ ├── CommandLineFixture.cs
│ └── SilentProcessRunner.cs
├── OctodiffAsync/
│ ├── CommandLine/
│ │ ├── DeltaCommand.cs
│ │ ├── ExplainDeltaCommand.cs
│ │ ├── HelpCommand.cs
│ │ ├── PatchCommand.cs
│ │ ├── SignatureCommand.cs
│ │ └── Support/
│ │ ├── CommandAttribute.cs
│ │ ├── CommandException.cs
│ │ ├── CommandLocator.cs
│ │ ├── ICommand.cs
│ │ ├── ICommandLocator.cs
│ │ ├── ICommandMetadata.cs
│ │ └── NDesk.Options.cs
│ ├── OctodiffAsync.csproj
│ ├── OctodiffAsyncProgram.cs
│ ├── Properties/
│ │ └── AssemblyInfo.cs
│ └── app.config
└── Tests.ps1
SYMBOL INDEX (611 symbols across 101 files)
FILE: source/FastRsync.BackwardCompatibilityTests/BackwardCompatibilityTests.cs
class BackwardCompatibilityTests (line 14) | [TestFixture]
method LegacyLibraryAppliesNewPatch (line 17) | [Test]
method LegacyLibraryPreparesAndAppliesNewPatch (line 42) | [Test]
method LegacyLibraryPreparesPatchForNewLibraryToApplyIt (line 68) | [Test]
method PrepareTestDataAsync (line 94) | private static async Task<(MemoryStream baseDataStream, MemoryStream b...
FILE: source/FastRsync.Benchmarks/BuildPatchBenchmark.cs
class BuildPatchBenchmark (line 10) | public class BuildPatchBenchmark
method GlobalSetup (line 28) | [GlobalSetup]
method BuildPatchXxHash (line 73) | [Benchmark]
method BuildPatchXxHash3 (line 85) | [Benchmark]
method BuildPatchSha1 (line 97) | [Benchmark]
method BuildPatchMd5 (line 109) | [Benchmark]
FILE: source/FastRsync.Benchmarks/HashBenchmark.cs
class HashBenchmark (line 12) | public class HashBenchmark
method GlobalSetup (line 32) | [GlobalSetup]
method DataXxHash64 (line 47) | [Benchmark]
method DataXxHash64MemoryStream (line 53) | [Benchmark]
method DataXxHash64AzureBlobStream (line 60) | [Benchmark]
method SystemXxHash64 (line 68) | [Benchmark]
method SystemXxHash64MemoryStream (line 74) | [Benchmark]
method SystemXxHash64AzureBlobStream (line 82) | [Benchmark]
method SystemXxHash64Reverse (line 92) | [Benchmark]
method SystemXxHash64AzureBlobStreamReverse (line 100) | [Benchmark]
FILE: source/FastRsync.Benchmarks/Program.cs
class Program (line 6) | class Program
method Main (line 8) | static void Main(string[] args)
FILE: source/FastRsync.Benchmarks/RollingChecksumBenchmark.cs
class RollingCheckSumBenchmark (line 8) | public class RollingCheckSumBenchmark
method GlobalSetup (line 23) | [GlobalSetup]
method Adler32RollingCalculateChecksum (line 33) | [Benchmark]
method Adler32RollingV2CalculateChecksum (line 39) | [Benchmark]
method Adler32RollingV3CalculateChecksum (line 45) | [Benchmark]
method Adler32RollingRotateChecksum (line 51) | [Benchmark]
method Adler32RollingV2RotateChecksum (line 57) | [Benchmark]
method Adler32RollingV3RotateChecksum (line 63) | [Benchmark]
FILE: source/FastRsync.Benchmarks/SignatureBenchmark.cs
class SignatureBenchmark (line 10) | public class SignatureBenchmark
method GlobalSetup (line 35) | [GlobalSetup]
method SignaturexxHash (line 49) | [Benchmark]
method SignaturexxHashAdler32V2 (line 58) | [Benchmark]
method SignaturexxHash3 (line 67) | [Benchmark]
method SignatureSha1 (line 76) | [Benchmark]
method SignatureMd5 (line 85) | [Benchmark]
method OctodiffSignaturexxHash (line 94) | [Benchmark]
FILE: source/FastRsync.Compression/GZip.cs
class GZip (line 29) | public class GZip
method Compress (line 37) | public static void Compress(Stream sourceStream, Stream destStream)
method CompressAsync (line 69) | public static Task CompressAsync(Stream sourceStream, Stream destStrea...
method CompressAsync (line 72) | public static async Task CompressAsync(Stream sourceStream, Stream des...
FILE: source/FastRsync.Tests/Adler32RollingChecksumAlgorithmsTests.cs
class Adler32RollingChecksumAlgorithmsTests (line 17) | [TestFixture]
method EveryRollingChecksumAlgorithmCalculatesRotateCorrectly (line 32) | [Test]
method SignDeltaAndApplyPatchTestCases (line 54) | public static IEnumerable<TestCaseData> SignDeltaAndApplyPatchTestCases()
method SignDeltaAndApplyPatchWithEveryRollingChecksumAlgorithm (line 63) | [Test]
method ModifyBytes (line 96) | private static byte[] ModifyBytes(ReadOnlySpan<byte> input, int percen...
method GetRandomBytes (line 119) | private static byte[] GetRandomBytes(int size)
FILE: source/FastRsync.Tests/Adler32RollingChecksumTests.cs
class Adler32RollingChecksumTests (line 6) | [TestFixture]
method Adler32RollingChecksum_CalculatesChecksum (line 9) | [Test]
FILE: source/FastRsync.Tests/Adler32RollingChecksumV2Tests.cs
class Adler32RollingChecksumV2Tests (line 6) | [TestFixture]
method Adler32RollingChecksumV2_CalculatesChecksum (line 9) | [Test]
FILE: source/FastRsync.Tests/Adler32RollingChecksumV3Tests.cs
class Adler32RollingChecksumV3Tests (line 7) | [TestFixture]
method Adler32RollingChecksumV3_CalculatesChecksum (line 10) | [Test]
method Adler32RollingChecksumV3_RotatesChecksum (line 36) | [Test]
FILE: source/FastRsync.Tests/BackwardCompatibilityTests.cs
class BackwardCompatibilityTests (line 14) | [TestFixture]
method ApplyOldPatch (line 17) | [Test]
method BuildPatchFromOldSignature (line 43) | [Test]
FILE: source/FastRsync.Tests/CommonAsserts.cs
class CommonAsserts (line 8) | class CommonAsserts
method ValidateSignature (line 10) | public static void ValidateSignature(Stream signatureStream, IHashAlgo...
FILE: source/FastRsync.Tests/DeltaReaderTests.cs
class DeltaReaderTests (line 10) | [TestFixture]
method BinaryDeltaReader_ReadsLegacyDelta (line 31) | [Test]
method LegacyBinaryDeltaReader_ReadsDelta (line 52) | [Test]
FILE: source/FastRsync.Tests/FastRsyncLegacy/BinaryDeltaReaderLegacy.cs
class BinaryFormat (line 17) | internal class BinaryFormat
class OctoBinaryFormat (line 26) | internal class OctoBinaryFormat
class FastRsyncBinaryFormat (line 35) | internal class FastRsyncBinaryFormat
class ProgressReport (line 43) | public sealed class ProgressReport
class BinaryDeltaReaderLegacy (line 52) | internal class BinaryDeltaReaderLegacy : IDeltaReaderLegacy
method BinaryDeltaReaderLegacy (line 60) | public BinaryDeltaReaderLegacy(Stream stream, IProgress<ProgressReport...
method ReadMetadata (line 97) | private void ReadMetadata()
method ReadFastRsyncDeltaHeader (line 121) | private void ReadFastRsyncDeltaHeader()
method ReadOctoDeltaHeader (line 136) | private void ReadOctoDeltaHeader()
method Apply (line 161) | public void Apply(
method ApplyAsync (line 200) | public async Task ApplyAsync(
FILE: source/FastRsync.Tests/FastRsyncLegacy/DeltaMetadataLegacy.cs
class DeltaMetadataLegacy (line 3) | internal class DeltaMetadataLegacy
FILE: source/FastRsync.Tests/FastRsyncLegacy/IDeltaReaderLegacy.cs
type IDeltaReaderLegacy (line 8) | internal interface IDeltaReaderLegacy
method Apply (line 14) | void Apply(
method ApplyAsync (line 19) | Task ApplyAsync(
FILE: source/FastRsync.Tests/FastRsyncLegacy/NewtonsoftJsonSerializationSettings.cs
class NewtonsoftJsonSerializationSettings (line 6) | public class NewtonsoftJsonSerializationSettings
method NewtonsoftJsonSerializationSettings (line 8) | static NewtonsoftJsonSerializationSettings()
FILE: source/FastRsync.Tests/GZipTests.cs
class GZipTests (line 15) | [TestFixture]
method GZip_CompressRandomData (line 18) | [Test]
method GZip_CompressData (line 30) | [Test]
method CompressData (line 43) | private static MemoryStream CompressData(byte[] data)
method GZip_CompressRandomData_RsyncSignatureAndPatch (line 65) | [Test]
method GZip_CompressData_RsyncSignatureAndPatch (line 86) | [Test]
method FullCompressedRsyncFlow (line 104) | private static MemoryStream FullCompressedRsyncFlow(MemoryStream basis...
method GZip_CompressRandomDataAsync_RsyncSignatureAndPatch (line 152) | [Test]
method GZip_CompressDataAsync_RsyncSignatureAndPatch (line 174) | [Test]
method FullCompressedRsyncFlowAsync (line 191) | private static async Task<MemoryStream> FullCompressedRsyncFlowAsync(M...
FILE: source/FastRsync.Tests/HashTests.cs
class HashTests (line 12) | [TestFixture]
method XxHash64StaticBackwardCompatibility (line 15) | [Test]
method XxHash64BackwardCompatibility (line 41) | [Test]
method XxHash64StreamBackwardCompatibility (line 71) | [Test]
FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffDeltaWriter.cs
type IOctodiffDeltaWriter (line 8) | public interface IOctodiffDeltaWriter
method WriteMetadata (line 10) | void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFil...
method WriteCopyCommand (line 11) | void WriteCopyCommand(DataRange segment);
method WriteDataCommand (line 12) | void WriteDataCommand(Stream source, long offset, long length);
method Finish (line 13) | void Finish();
FILE: source/FastRsync.Tests/OctodiffLegacy/IOctodiffSignatureWriter.cs
type IOctodiffSignatureWriter (line 7) | public interface IOctodiffSignatureWriter
method WriteMetadata (line 9) | void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum roll...
method WriteChunk (line 10) | void WriteChunk(ChunkSignature signature);
class OctodiffSignatureWriter (line 13) | public class OctodiffSignatureWriter(Stream signatureStream) : IOctodiff...
method WriteMetadata (line 17) | public void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecks...
method WriteChunk (line 26) | public void WriteChunk(ChunkSignature signature)
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffAggregateCopyOperationsDecorator.cs
class OctodiffAggregateCopyOperationsDecorator (line 15) | public class OctodiffAggregateCopyOperationsDecorator : IOctodiffDeltaWr...
method OctodiffAggregateCopyOperationsDecorator (line 20) | public OctodiffAggregateCopyOperationsDecorator(IOctodiffDeltaWriter d...
method WriteDataCommand (line 25) | public void WriteDataCommand(Stream source, long offset, long length)
method WriteMetadata (line 31) | public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expecte...
method WriteCopyCommand (line 36) | public void WriteCopyCommand(DataRange chunk)
method FlushCurrentCopyCommand (line 49) | private void FlushCurrentCopyCommand()
method Finish (line 60) | public void Finish()
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryDeltaWriter.cs
class OctodiffBinaryDeltaWriter (line 8) | public class OctodiffBinaryDeltaWriter : IOctodiffDeltaWriter
method OctodiffBinaryDeltaWriter (line 13) | public OctodiffBinaryDeltaWriter(Stream stream, int readWriteBufferSiz...
method WriteMetadata (line 19) | public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expecte...
method WriteCopyCommand (line 29) | public void WriteCopyCommand(DataRange segment)
method WriteDataCommand (line 36) | public void WriteDataCommand(Stream source, long offset, long length)
method Finish (line 62) | public void Finish()
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryFormat.cs
class OctodiffBinaryFormat (line 5) | class OctodiffBinaryFormat
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffDeltaBuilder.cs
class OctodiffDeltaBuilder (line 9) | public class OctodiffDeltaBuilder
method OctodiffDeltaBuilder (line 13) | public OctodiffDeltaBuilder(int readBufferSize = 4 * 1024 * 1024)
method BuildDelta (line 18) | public void BuildDelta(Stream newFileStream, IOctodiffSignatureReader ...
class ChunkSignatureChecksumComparer (line 114) | private class ChunkSignatureChecksumComparer : IComparer<ChunkSignature>
method Compare (line 116) | public int Compare(ChunkSignature x, ChunkSignature y)
method OrderChunksByChecksum (line 123) | private static List<ChunkSignature> OrderChunksByChecksum(List<ChunkSi...
method CreateChunkMap (line 129) | private Dictionary<uint, int> CreateChunkMap(IList<ChunkSignature> chu...
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignature.cs
class OctodiffSignature (line 7) | public class OctodiffSignature
method OctodiffSignature (line 9) | public OctodiffSignature(IHashAlgorithm hashAlgorithm, IRollingChecksu...
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureBuilder.cs
class OctodiffSignatureBuilder (line 8) | public class OctodiffSignatureBuilder
method OctodiffSignatureBuilder (line 16) | public OctodiffSignatureBuilder() : this(SupportedAlgorithms.Hashing.S...
method OctodiffSignatureBuilder (line 20) | public OctodiffSignatureBuilder(IHashAlgorithm hashAlgorithm, IRolling...
method Build (line 44) | public void Build(Stream stream, IOctodiffSignatureWriter signatureWri...
method WriteMetadata (line 50) | private void WriteMetadata(Stream stream, IOctodiffSignatureWriter sig...
method WriteChunkSignatures (line 56) | private void WriteChunkSignatures(Stream stream, IOctodiffSignatureWri...
FILE: source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureReader.cs
type IOctodiffSignatureReader (line 9) | public interface IOctodiffSignatureReader
method ReadSignature (line 11) | OctodiffSignature ReadSignature();
class OctodiffSignatureReader (line 14) | public class OctodiffSignatureReader : IOctodiffSignatureReader
method OctodiffSignatureReader (line 18) | public OctodiffSignatureReader(Stream stream, IProgress<ProgressReport...
method ReadSignature (line 23) | public OctodiffSignature ReadSignature()
FILE: source/FastRsync.Tests/PatchingAsyncTests.cs
class PatchingAsyncTests (line 16) | [TestFixture]
method PatchingAsyncXXHash_ForNewData_PatchesFile (line 19) | [Test]
method PatchingAsyncXXHash_ForOctodiffSignature_PatchesFile (line 71) | [Test]
method PatchingAsyncXXHash_ForOctodiffSignatureAndPatch_PatchesFile (line 103) | [Test]
method PatchingAsyncXXHash_ForTheSameData_PatchesFile (line 135) | [Test]
method PrepareTestDataAsync (line 171) | private static async Task<(MemoryStream baseDataStream, MemoryStream b...
method PrepareTestDataWithOctodiffSignature (line 192) | private static (MemoryStream baseDataStream, MemoryStream baseSignatur...
FILE: source/FastRsync.Tests/PatchingBigFilesTests.cs
class PatchingBigFilesTests (line 14) | [TestFixture]
method PatchingSyncXXHash_BigFile (line 17) | [Test]
method PrepareTestData (line 89) | public static Stream PrepareTestData(Stream baseDataStream, IHashAlgor...
method CompareFilesByHash (line 99) | public static bool CompareFilesByHash(string fileName1, string fileName2)
FILE: source/FastRsync.Tests/PatchingSyncTests.cs
class PatchingSyncTests (line 14) | [TestFixture]
method PatchingSyncXXHash_ForNewData_PatchesFile (line 17) | [Test]
method PatchingSyncXXHash_ForOctodiffSignature_PatchesFile (line 49) | [Test]
method PatchingSyncXXHash_ForOctodiffSignatureAndPatch_PatchesFile (line 81) | [Test]
method PrepareTestDataWithOctodiffSignature (line 113) | private static (MemoryStream baseDataStream, MemoryStream baseSignatur...
FILE: source/FastRsync.Tests/SignatureBuilderAsyncRandomDataTests.cs
class SignatureBuilderAsyncRandomDataTests (line 14) | [TestFixture]
method SignatureBuilderAsyncXXHash_ForRandomData_BuildsSignature (line 17) | [Test]
method SignatureBuilderAsyncXXHashAdlerV3_ForRandomData_BuildsSignature (line 51) | [Test]
method SignatureBuilderAsyncSha1_ForRandomData_BuildsSignature (line 85) | [Test]
FILE: source/FastRsync.Tests/SignatureBuilderSyncRandomDataTests.cs
class SignatureBuilderSyncRandomDataTests (line 13) | [TestFixture]
method SignatureBuilderXXHash_ForRandomData_BuildsSignature (line 16) | [Test]
method SignatureBuilderXXHashAdlerV3_ForRandomData_BuildsSignature (line 50) | [Test]
method SignatureBuilderSha1_ForRandomData_BuildsSignature (line 84) | [Test]
FILE: source/FastRsync.Tests/SignatureBuilderTests.cs
class SignatureBuilderTests (line 14) | [TestFixture]
method SignatureBuilderXXHash_BuildsSignature (line 23) | [Test]
method SignatureBuilderAsyncXXHash_BuildsSignature (line 50) | [Test]
method SignatureBuilderAsyncXXHash_ForEmptyStream_BuildsSignature (line 77) | [Test]
method SignatureBuilderSyncXXHash_ForEmptyStream_BuildsSignature (line 99) | [Test]
FILE: source/FastRsync.Tests/SignatureReaderTests.cs
class SignatureReaderTests (line 12) | [TestFixture]
method SignatureReader_ReadsOctodiffSignature (line 15) | [Test]
method SignatureReader_ReadsRandomData_ThrowsException (line 38) | [Test]
FILE: source/FastRsync.Tests/Utils.cs
class Utils (line 8) | class Utils
method GetMd5 (line 10) | public static string GetMd5(byte[] data)
method PrepareTestData (line 16) | public static (MemoryStream baseDataStream, MemoryStream baseSignature...
FILE: source/FastRsync/Core/AggregateCopyOperationsDecorator.cs
class AggregateCopyOperationsDecorator (line 16) | public class AggregateCopyOperationsDecorator : IDeltaWriter
method AggregateCopyOperationsDecorator (line 21) | public AggregateCopyOperationsDecorator(IDeltaWriter decorated)
method WriteDataCommand (line 26) | public void WriteDataCommand(Stream source, long offset, long length)
method WriteDataCommandAsync (line 32) | public async Task WriteDataCommandAsync(Stream source, long offset, lo...
method WriteMetadata (line 38) | public void WriteMetadata(DeltaMetadata metadata)
method WriteCopyCommand (line 43) | public void WriteCopyCommand(DataRange chunk)
method FlushCurrentCopyCommand (line 56) | private void FlushCurrentCopyCommand()
method Finish (line 67) | public void Finish()
FILE: source/FastRsync/Core/BinaryFormat.cs
class BinaryFormat (line 5) | internal class BinaryFormat
class OctoBinaryFormat (line 14) | internal class OctoBinaryFormat
class FastRsyncBinaryFormat (line 23) | internal class FastRsyncBinaryFormat
FILE: source/FastRsync/Core/ChunkSignature.cs
class ChunkSignature (line 5) | public class ChunkSignature
method ToString (line 12) | public override string ToString()
FILE: source/FastRsync/Core/ChunkSignatureChecksumComparer.cs
class ChunkSignatureChecksumComparer (line 5) | class ChunkSignatureChecksumComparer : IComparer<ChunkSignature>
method Compare (line 7) | public int Compare(ChunkSignature x, ChunkSignature y)
FILE: source/FastRsync/Core/JsonContext.cs
class JsonContextCore (line 12) | [JsonSerializable(typeof(DeltaMetadata))]
FILE: source/FastRsync/Core/JsonSerializationSettings.cs
class JsonSerializationSettings (line 6) | public class JsonSerializationSettings
method JsonSerializationSettings (line 8) | static JsonSerializationSettings()
FILE: source/FastRsync/Core/SupportedAlgorithms.cs
class SupportedAlgorithms (line 8) | public static class SupportedAlgorithms
class Hashing (line 10) | public static class Hashing
method Sha1 (line 12) | public static IHashAlgorithm Sha1()
method Md5 (line 17) | public static IHashAlgorithm Md5()
method XxHash (line 22) | public static IHashAlgorithm XxHash()
method XxHash3 (line 27) | public static IHashAlgorithm XxHash3()
method Default (line 32) | public static IHashAlgorithm Default()
method Create (line 37) | public static IHashAlgorithm Create(string algorithmName)
class Checksum (line 55) | public static class Checksum
method Adler32Rolling (line 57) | public static IRollingChecksum Adler32Rolling() { return new Adler32...
method Adler32RollingV2 (line 58) | [Obsolete("Adler32V2 has buggy mod operation implemented. See https:...
method Adler32RollingV3 (line 61) | public static IRollingChecksum Adler32RollingV3() { return new Adler...
method Default (line 63) | public static IRollingChecksum Default()
method Create (line 68) | public static IRollingChecksum Create(string algorithm)
FILE: source/FastRsync/Delta/BinaryDeltaReader.cs
class BinaryDeltaReader (line 14) | public class BinaryDeltaReader : IDeltaReader
method BinaryDeltaReader (line 22) | public BinaryDeltaReader(Stream stream, IProgress<ProgressReport> prog...
method ReadMetadata (line 69) | private void ReadMetadata()
method ReadFastRsyncDeltaHeader (line 93) | private void ReadFastRsyncDeltaHeader()
method ReadOctoDeltaHeader (line 111) | private void ReadOctoDeltaHeader()
method Apply (line 136) | public void Apply(
method ApplyAsync (line 175) | public Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, ...
method ApplyAsync (line 178) | public async Task ApplyAsync(Func<byte[], Task> writeData, Func<long, ...
FILE: source/FastRsync/Delta/BinaryDeltaWriter.cs
class BinaryDeltaWriter (line 10) | public class BinaryDeltaWriter : IDeltaWriter
method BinaryDeltaWriter (line 16) | public BinaryDeltaWriter(Stream stream, int readWriteBufferSize = 1024...
method WriteMetadata (line 23) | public void WriteMetadata(DeltaMetadata metadata)
method WriteCopyCommand (line 35) | public void WriteCopyCommand(DataRange segment)
method WriteDataCommand (line 42) | public void WriteDataCommand(Stream source, long offset, long length)
method WriteDataCommandAsync (line 68) | public async Task WriteDataCommandAsync(Stream source, long offset, lo...
method Finish (line 97) | public void Finish()
FILE: source/FastRsync/Delta/DataRange.cs
type DataRange (line 3) | public struct DataRange
method DataRange (line 5) | public DataRange(long startOffset, long length)
FILE: source/FastRsync/Delta/DeltaApplier.cs
class DeltaApplier (line 10) | public class DeltaApplier
method DeltaApplier (line 14) | public DeltaApplier(int readBufferSize = 4 * 1024 * 1024)
method Apply (line 22) | public void Apply(Stream basisFileStream, IDeltaReader delta, Stream o...
method ApplyAsync (line 51) | public Task ApplyAsync(Stream basisFileStream, IDeltaReader delta, Str...
method ApplyAsync (line 54) | public async Task ApplyAsync(Stream basisFileStream, IDeltaReader delt...
method HashCheck (line 83) | public bool HashCheck(IDeltaReader delta, Stream outputStream)
method HashCheckAsync (line 95) | public Task<bool> HashCheckAsync(IDeltaReader delta, Stream outputStre...
method HashCheckAsync (line 97) | public async Task<bool> HashCheckAsync(IDeltaReader delta, Stream outp...
FILE: source/FastRsync/Delta/DeltaBuilder.cs
class DeltaBuilder (line 13) | public class DeltaBuilder
method DeltaBuilder (line 17) | public DeltaBuilder(int readBufferSize = 4 * 1024 * 1024)
method BuildDelta (line 25) | public void BuildDelta(Stream newFileStream, ISignatureReader signatur...
method BuildDeltaAsync (line 141) | public Task BuildDeltaAsync(Stream newFileStream, ISignatureReader sig...
method BuildDeltaAsync (line 144) | public async Task BuildDeltaAsync(Stream newFileStream, ISignatureRead...
method OrderChunksByChecksum (line 260) | private static List<ChunkSignature> OrderChunksByChecksum(List<ChunkSi...
method CreateChunkMap (line 266) | private Dictionary<uint, int> CreateChunkMap(IList<ChunkSignature> chu...
FILE: source/FastRsync/Delta/DeltaMetadata.cs
class DeltaMetadata (line 3) | public class DeltaMetadata
FILE: source/FastRsync/Delta/IDeltaReader.cs
type IDeltaReader (line 9) | public interface IDeltaReader
method Apply (line 15) | void Apply(Action<byte[]> writeData, Action<long, long> copy);
method ApplyAsync (line 16) | Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> c...
method ApplyAsync (line 17) | Task ApplyAsync(Func<byte[], Task> writeData, Func<long, long, Task> c...
FILE: source/FastRsync/Delta/IDeltaWriter.cs
type IDeltaWriter (line 8) | public interface IDeltaWriter
method WriteMetadata (line 10) | void WriteMetadata(DeltaMetadata metadata);
method WriteCopyCommand (line 11) | void WriteCopyCommand(DataRange segment);
method WriteDataCommand (line 12) | void WriteDataCommand(Stream source, long offset, long length);
method WriteDataCommandAsync (line 13) | Task WriteDataCommandAsync(Stream source, long offset, long length, Ca...
method Finish (line 14) | void Finish();
FILE: source/FastRsync/Diagnostics/ConsoleProgressReporter.cs
class ConsoleProgressReporter (line 5) | public class ConsoleProgressReporter : IProgress<ProgressReport>
method Report (line 10) | public void Report(ProgressReport progress)
FILE: source/FastRsync/Diagnostics/ProgressReport.cs
type ProgressOperationType (line 3) | public enum ProgressOperationType
class ProgressReport (line 13) | public sealed class ProgressReport
FILE: source/FastRsync/Hash/Adler32RollingChecksum.cs
class Adler32RollingChecksum (line 5) | public class Adler32RollingChecksum : IRollingChecksum
method Calculate (line 9) | public UInt32 Calculate(byte[] block, int offset, int count)
method Rotate (line 22) | public UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunk...
FILE: source/FastRsync/Hash/Adler32RollingChecksumV2.cs
class Adler32RollingChecksumV2 (line 5) | [Obsolete("Adler32V2 has buggy mod operation implemented. See https://gi...
method Calculate (line 12) | public uint Calculate(byte[] block, int offset, int count)
method Rotate (line 25) | public uint Rotate(uint checksum, byte remove, byte add, int chunkSize)
FILE: source/FastRsync/Hash/Adler32RollingChecksumV3.cs
class Adler32RollingChecksumV3 (line 6) | public class Adler32RollingChecksumV3 : IRollingChecksum
method Calculate (line 12) | public uint Calculate(byte[] block, int offset, int count)
method Rotate (line 27) | public uint Rotate(uint checksum, byte remove, byte add, int chunkSize)
FILE: source/FastRsync/Hash/CryptographyHashAlgorithmWrapper.cs
class CryptographyHashAlgorithmWrapper (line 8) | public class CryptographyHashAlgorithmWrapper : IHashAlgorithm
method CryptographyHashAlgorithmWrapper (line 12) | public CryptographyHashAlgorithmWrapper(string name, HashAlgorithm alg...
method ComputeHash (line 21) | public byte[] ComputeHash(Stream stream)
method ComputeHashAsync (line 26) | public async Task<byte[]> ComputeHashAsync(Stream stream, Cancellation...
method ComputeHash (line 35) | public byte[] ComputeHash(byte[] buffer, int offset, int length)
FILE: source/FastRsync/Hash/IHashAlgorithm.cs
type IHashAlgorithm (line 7) | public interface IHashAlgorithm
method ComputeHash (line 11) | byte[] ComputeHash(Stream stream);
method ComputeHashAsync (line 12) | Task<byte[]> ComputeHashAsync(Stream stream, CancellationToken cancell...
method ComputeHash (line 13) | byte[] ComputeHash(byte[] buffer, int offset, int length);
FILE: source/FastRsync/Hash/IRollingChecksum.cs
type IRollingChecksum (line 5) | public interface IRollingChecksum
method Calculate (line 8) | UInt32 Calculate(byte[] block, int offset, int count);
method Rotate (line 9) | UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize);
FILE: source/FastRsync/Hash/NonCryptographicHashAlgorithmWrapper.cs
class NonCryptographicHashAlgorithmWrapper (line 9) | public class NonCryptographicHashAlgorithmWrapper : IHashAlgorithm
method NonCryptographicHashAlgorithmWrapper (line 17) | public NonCryptographicHashAlgorithmWrapper(string name, NonCryptograp...
method ComputeHash (line 23) | public byte[] ComputeHash(Stream stream)
method ComputeHashAsync (line 32) | public async Task<byte[]> ComputeHashAsync(Stream stream, Cancellation...
method ComputeHash (line 41) | public byte[] ComputeHash(byte[] buffer, int offset, int length)
FILE: source/FastRsync/Signature/ISignatureReader.cs
type ISignatureReader (line 3) | public interface ISignatureReader
method ReadSignature (line 5) | Signature ReadSignature();
method ReadSignatureMetadata (line 6) | Signature ReadSignatureMetadata();
FILE: source/FastRsync/Signature/ISignatureWriter.cs
type ISignatureWriter (line 7) | public interface ISignatureWriter
method WriteMetadata (line 9) | void WriteMetadata(SignatureMetadata metadata);
method WriteMetadataAsync (line 10) | Task WriteMetadataAsync(SignatureMetadata metadata, CancellationToken ...
method WriteChunk (line 11) | void WriteChunk(ChunkSignature signature);
method WriteChunkAsync (line 12) | Task WriteChunkAsync(ChunkSignature signature, CancellationToken cance...
FILE: source/FastRsync/Signature/Signature.cs
class SignatureMetadata (line 7) | public class SignatureMetadata
type RsyncFormatType (line 15) | public enum RsyncFormatType
class Signature (line 21) | public class Signature
method Signature (line 23) | public Signature(SignatureMetadata metadata, RsyncFormatType type)
FILE: source/FastRsync/Signature/SignatureBuilder.cs
class SignatureBuilder (line 11) | public class SignatureBuilder
method SignatureBuilder (line 19) | public SignatureBuilder() : this(SupportedAlgorithms.Hashing.Default()...
method SignatureBuilder (line 23) | public SignatureBuilder(IHashAlgorithm hashAlgorithm, IRollingChecksum...
method Build (line 50) | public void Build(Stream baseDataStream, ISignatureWriter signatureWri...
method BuildAsync (line 56) | public Task BuildAsync(Stream baseDataStream, ISignatureWriter signatu...
method BuildAsync (line 59) | public async Task BuildAsync(Stream baseDataStream, ISignatureWriter s...
method WriteMetadata (line 65) | private void WriteMetadata(Stream baseFileStream, ISignatureWriter sig...
method WriteMetadataAsync (line 94) | private async Task WriteMetadataAsync(Stream baseFileStream, ISignatur...
method WriteChunkSignatures (line 123) | private void WriteChunkSignatures(Stream baseFileStream, ISignatureWri...
method WriteChunkSignaturesAsync (line 159) | private async Task WriteChunkSignaturesAsync(Stream baseFileStream, IS...
FILE: source/FastRsync/Signature/SignatureReader.cs
class SignatureReader (line 10) | public class SignatureReader : ISignatureReader
method SignatureReader (line 15) | public SignatureReader(Stream stream, IProgress<ProgressReport> progre...
method ReadSignature (line 21) | public Signature ReadSignature()
method ReadSignatureMetadata (line 29) | public Signature ReadSignatureMetadata()
method ReadFastRsyncSignatureHeader (line 47) | private Signature ReadFastRsyncSignatureHeader()
method ReadOctoSignatureHeader (line 66) | private Signature ReadOctoSignatureHeader()
method ReadChunks (line 93) | private void ReadChunks(Signature signature)
method Progress (line 125) | private void Progress()
FILE: source/FastRsync/Signature/SignatureWriter.cs
class SignatureWriter (line 9) | public class SignatureWriter : ISignatureWriter
method SignatureWriter (line 14) | public SignatureWriter(Stream signatureStream)
method WriteMetadataInternal (line 20) | private static void WriteMetadataInternal(BinaryWriter bw, SignatureMe...
method WriteMetadata (line 33) | public void WriteMetadata(SignatureMetadata metadata)
method WriteMetadataAsync (line 38) | public async Task WriteMetadataAsync(SignatureMetadata metadata, Cance...
method WriteChunk (line 51) | public void WriteChunk(ChunkSignature signature)
method WriteChunkAsync (line 58) | public async Task WriteChunkAsync(ChunkSignature signature, Cancellati...
FILE: source/Octodiff.Tests/DeltaFixture.cs
class DeltaFixture (line 11) | [TestFixture]
method DeltaOfUnchangedFileShouldResultInJustCopySegment (line 14) | [Test]
method DeltaOfChangedFileShouldResultInNewDataSegments (line 36) | [Test]
FILE: source/Octodiff.Tests/HelpFixture.cs
class HelpFixture (line 6) | [TestFixture]
method ShouldPrintGeneralHelp (line 9) | [Test]
method ShouldPrintCommandHelp (line 21) | [Test]
method ShouldPrintHelpWhenAllArgumentsAreNotSpecified (line 35) | [Test]
FILE: source/Octodiff.Tests/PackageGenerator.cs
class PackageGenerator (line 9) | public class PackageGenerator
method GeneratePackage (line 13) | public static void GeneratePackage(string fileName, int numberOfFiles ...
method ModifyPackage (line 31) | public static void ModifyPackage(string fileName, string newFileName, ...
FILE: source/Octodiff.Tests/PatchFixture.cs
class PatchFixture (line 9) | [TestFixture]
method PatchingShouldResultInPerfectCopy (line 12) | [Test]
method PatchVerificationShouldFailWhenFilesModified (line 34) | [Test]
method PatchVerificationCanBeSkipped (line 55) | [Test]
method Sha1 (line 74) | static string Sha1(string fileName)
FILE: source/Octodiff.Tests/SignatureFixture.cs
class SignatureFixture (line 8) | [TestFixture]
method ShouldCreateSignature (line 11) | [Test]
method ShouldCreateDifferentSignaturesBasedOnChunkSize (line 35) | [Test]
method Length (line 62) | static long Length(string fileName)
FILE: source/Octodiff.Tests/Timings.cs
class TimingsFixture (line 9) | [TestFixture]
method ExecuteWithTimings (line 12) | [Test]
method Time (line 32) | static void Time(string task, Action callback)
FILE: source/Octodiff.Tests/Util/CommandLineFixture.cs
type OctodiffAppVariant (line 10) | public enum OctodiffAppVariant
class CommandLineFixture (line 16) | public abstract class CommandLineFixture
method Run (line 23) | public void Run(string args, OctodiffAppVariant octodiff)
FILE: source/Octodiff.Tests/Util/SilentProcessRunner.cs
class SilentProcessRunner (line 9) | public static class SilentProcessRunner
method SilentProcessRunner (line 15) | static SilentProcessRunner()
method ExecuteCommand (line 35) | public static int ExecuteCommand(string executable, string arguments, ...
method GetCPInfoEx (line 99) | [DllImport("kernel32.dll", SetLastError = true)]
type CPINFOEX (line 106) | [StructLayout(LayoutKind.Sequential)]
FILE: source/Octodiff/CommandLine/DeltaCommand.cs
class DeltaCommand (line 12) | [Command("delta", Description = "Given a signature file and a new file, ...
method DeltaCommand (line 21) | public DeltaCommand()
method GetHelp (line 30) | public void GetHelp(TextWriter writer)
method Execute (line 35) | public int Execute(string[] commandLineArguments)
FILE: source/Octodiff/CommandLine/ExplainDeltaCommand.cs
class ExplainDeltaCommand (line 9) | [Command("explain-delta", Description = "Prints instructions from a delt...
method ExplainDeltaCommand (line 15) | public ExplainDeltaCommand()
method GetHelp (line 21) | public void GetHelp(TextWriter writer)
method Execute (line 26) | public int Execute(string[] commandLineArguments)
FILE: source/Octodiff/CommandLine/HelpCommand.cs
class HelpCommand (line 8) | [Command("help", "?", "h", Description = "Prints this help text")]
method GetHelp (line 13) | public void GetHelp(TextWriter writer)
method Execute (line 17) | public int Execute(string[] commandLineArguments)
method PrintCommandHelp (line 47) | void PrintCommandHelp(string executable, ICommand command, ICommandMet...
method PrintGeneralHelp (line 60) | void PrintGeneralHelp(string executable)
FILE: source/Octodiff/CommandLine/PatchCommand.cs
class PatchCommand (line 9) | [Command("patch", Description = "Given a basis file, and a delta, produc...
method PatchCommand (line 19) | public PatchCommand()
method GetHelp (line 29) | public void GetHelp(TextWriter writer)
method Execute (line 34) | public int Execute(string[] commandLineArguments)
FILE: source/Octodiff/CommandLine/SignatureCommand.cs
class SignatureCommand (line 10) | [Command("signature", "sig", Description = "Given a basis file, creates ...
method SignatureCommand (line 18) | public SignatureCommand()
method GetHelp (line 27) | public void GetHelp(TextWriter writer)
method Execute (line 32) | public int Execute(string[] commandLineArguments)
FILE: source/Octodiff/CommandLine/Support/CommandAttribute.cs
class CommandAttribute (line 5) | [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited...
method CommandAttribute (line 8) | public CommandAttribute(string name, params string[] aliases)
FILE: source/Octodiff/CommandLine/Support/CommandException.cs
class CommandException (line 5) | class CommandException : Exception
method CommandException (line 7) | public CommandException(string message)
FILE: source/Octodiff/CommandLine/Support/CommandLocator.cs
class CommandLocator (line 6) | class CommandLocator : ICommandLocator
method List (line 8) | public ICommandMetadata[] List()
method Find (line 18) | public ICommandMetadata Find(string name)
method Create (line 29) | public ICommand Create(ICommandMetadata metadata)
FILE: source/Octodiff/CommandLine/Support/ICommand.cs
type ICommand (line 5) | interface ICommand
method GetHelp (line 7) | void GetHelp(TextWriter writer);
method Execute (line 8) | int Execute(string[] commandLineArguments);
FILE: source/Octodiff/CommandLine/Support/ICommandLocator.cs
type ICommandLocator (line 3) | interface ICommandLocator
method List (line 5) | ICommandMetadata[] List();
method Find (line 6) | ICommandMetadata Find(string name);
method Create (line 7) | ICommand Create(ICommandMetadata metadata);
FILE: source/Octodiff/CommandLine/Support/ICommandMetadata.cs
type ICommandMetadata (line 3) | interface ICommandMetadata
FILE: source/Octodiff/CommandLine/Support/NDesk.Options.cs
class OptionValueCollection (line 152) | class OptionValueCollection : IList, IList<string>
method OptionValueCollection (line 158) | internal OptionValueCollection(OptionContext c)
method CopyTo (line 164) | void ICollection.CopyTo(Array array, int index) { (values as ICollecti...
method Add (line 170) | public void Add(string item) { values.Add(item); }
method Clear (line 171) | public void Clear() { values.Clear(); }
method Contains (line 172) | public bool Contains(string item) { return values.Contains(item); }
method CopyTo (line 173) | public void CopyTo(string[] array, int arrayIndex) { values.CopyTo(arr...
method Remove (line 174) | public bool Remove(string item) { return values.Remove(item); }
method GetEnumerator (line 180) | IEnumerator IEnumerable.GetEnumerator() { return values.GetEnumerator(...
method GetEnumerator (line 184) | public IEnumerator<string> GetEnumerator() { return values.GetEnumerat...
method Add (line 188) | int IList.Add(object value) { return (values as IList).Add(value); }
method Contains (line 189) | bool IList.Contains(object value) { return (values as IList).Contains(...
method IndexOf (line 190) | int IList.IndexOf(object value) { return (values as IList).IndexOf(val...
method Insert (line 191) | void IList.Insert(int index, object value) { (values as IList).Insert(...
method Remove (line 192) | void IList.Remove(object value) { (values as IList).Remove(value); }
method RemoveAt (line 193) | void IList.RemoveAt(int index) { (values as IList).RemoveAt(index); }
method IndexOf (line 199) | public int IndexOf(string item) { return values.IndexOf(item); }
method Insert (line 200) | public void Insert(int index, string item) { values.Insert(index, item...
method RemoveAt (line 201) | public void RemoveAt(int index) { values.RemoveAt(index); }
method AssertValid (line 203) | private void AssertValid(int index)
method ToList (line 230) | public List<string> ToList()
method ToArray (line 235) | public string[] ToArray()
method ToString (line 240) | public override string ToString()
class OptionContext (line 246) | class OptionContext
method OptionContext (line 254) | public OptionContext(OptionSet set)
type OptionValueType (line 289) | enum OptionValueType
class Option (line 296) | abstract class Option
method Option (line 304) | protected Option(string prototype, string description)
method Option (line 309) | protected Option(string prototype, string description, int maxValueCount)
method GetNames (line 346) | public string[] GetNames()
method GetValueSeparators (line 351) | public string[] GetValueSeparators()
method Parse (line 358) | protected static T Parse<T>(string value, OptionContext c)
method ParsePrototype (line 388) | private OptionValueType ParsePrototype()
method AddSeparators (line 431) | private static void AddSeparators(string name, int end, ICollection<st...
method Invoke (line 465) | public void Invoke(OptionContext c)
method OnParseComplete (line 473) | protected abstract void OnParseComplete(OptionContext c);
method ToString (line 475) | public override string ToString()
class OptionException (line 481) | [Serializable]
method OptionException (line 486) | public OptionException()
method OptionException (line 490) | public OptionException(string message, string optionName)
method OptionException (line 496) | public OptionException(string message, string optionName, Exception in...
method OptionException (line 502) | protected OptionException(SerializationInfo info, StreamingContext con...
method GetObjectData (line 513) | [SecurityPermission(SecurityAction.LinkDemand, SerializationFormatter ...
class OptionSet (line 523) | class OptionSet : KeyedCollection<string, Option>
method OptionSet (line 527) | public OptionSet()
method OptionSet (line 532) | public OptionSet(Converter<string, string> localizer)
method GetKeyForItem (line 544) | protected override string GetKeyForItem(Option item)
method GetOptionForName (line 555) | [Obsolete("Use KeyedCollection.this[string]")]
method InsertItem (line 570) | protected override void InsertItem(int index, Option item)
method RemoveItem (line 576) | protected override void RemoveItem(int index)
method SetItem (line 587) | protected override void SetItem(int index, Option item)
method AddImpl (line 594) | private void AddImpl(Option option)
method Add (line 616) | public new OptionSet Add(Option option)
method Positional (line 622) | public void Positional(string name, string description, Action<string>...
class ActionOption (line 627) | sealed class ActionOption : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
method Add (line 645) | public OptionSet Add(string prototype, Action<string> action)
method Add (line 650) | public OptionSet Add(string prototype, string description, Action<stri...
method Add (line 660) | public OptionSet Add(string prototype, OptionAction<string, string> ac...
method Add (line 665) | public OptionSet Add(string prototype, string description, OptionActio...
class ActionOption (line 675) | sealed class ActionOption<T> : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
class ActionOption (line 693) | sealed class ActionOption<TKey, TValue> : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
method Add (line 713) | public OptionSet Add<T>(string prototype, Action<T> action)
method Add (line 718) | public OptionSet Add<T>(string prototype, string description, Action<T...
method Add (line 723) | public OptionSet Add<TKey, TValue>(string prototype, OptionAction<TKey...
method Add (line 728) | public OptionSet Add<TKey, TValue>(string prototype, string descriptio...
method CreateOptionContext (line 733) | protected virtual OptionContext CreateOptionContext()
method Parse (line 739) | public List<string> Parse (IEnumerable<string> arguments)
method Parse (line 767) | public List<string> Parse(IEnumerable<string> arguments)
method Unprocessed (line 805) | private static bool Unprocessed(ICollection<string> extra, Option def,...
method GetOptionParts (line 821) | protected bool GetOptionParts(string argument, out string flag, out st...
method Parse (line 842) | protected virtual bool Parse(string argument, OptionContext c)
method ParseValue (line 883) | private void ParseValue(string option, OptionContext c)
method ParseBool (line 904) | private bool ParseBool(string option, string n, OptionContext c)
method ParseBundledValue (line 922) | private bool ParseBundledValue(string f, string n, OptionContext c)
method Invoke (line 960) | private static void Invoke(OptionContext c, string name, string value,...
method GetPositionals (line 970) | public List<string> GetPositionals()
method WriteOptionDescriptions (line 975) | public void WriteOptionDescriptions(TextWriter o)
method WriteOptionPrototype (line 1044) | bool WriteOptionPrototype(TextWriter o, Option p, ref int written)
method GetNextOptionIndex (line 1094) | static int GetNextOptionIndex(string[] names, int i)
method Write (line 1103) | static void Write(TextWriter o, ref int n, string s)
method GetArgumentName (line 1109) | private static string GetArgumentName(int index, int maxIndex, string ...
method GetDescription (line 1135) | private static string GetDescription(string description)
method GetLines (line 1182) | private static IEnumerable<string> GetLines(string description)
method IsEolChar (line 1208) | private static bool IsEolChar(char c)
method GetLineEnd (line 1213) | private static int GetLineEnd(int start, int length, string description)
FILE: source/Octodiff/OctodiffProgram.cs
class OctodiffProgram (line 10) | public class OctodiffProgram
method Main (line 12) | static int Main(string[] args)
method WriteError (line 63) | static void WriteError(Exception ex, bool details = false)
method ExtractCommand (line 74) | private static string ExtractCommand(ICollection<string> args, out str...
FILE: source/OctodiffAsync/CommandLine/DeltaCommand.cs
class DeltaCommand (line 12) | [Command("delta", Description = "Given a signature file and a new file, ...
method DeltaCommand (line 21) | public DeltaCommand()
method GetHelp (line 30) | public void GetHelp(TextWriter writer)
method Execute (line 35) | public int Execute(string[] commandLineArguments)
FILE: source/OctodiffAsync/CommandLine/ExplainDeltaCommand.cs
class ExplainDeltaCommand (line 10) | [Command("explain-delta", Description = "Prints instructions from a delt...
method ExplainDeltaCommand (line 16) | public ExplainDeltaCommand()
method GetHelp (line 22) | public void GetHelp(TextWriter writer)
method Execute (line 27) | public int Execute(string[] commandLineArguments)
FILE: source/OctodiffAsync/CommandLine/HelpCommand.cs
class HelpCommand (line 8) | [Command("help", "?", "h", Description = "Prints this help text")]
method GetHelp (line 13) | public void GetHelp(TextWriter writer)
method Execute (line 17) | public int Execute(string[] commandLineArguments)
method PrintCommandHelp (line 47) | void PrintCommandHelp(string executable, ICommand command, ICommandMet...
method PrintGeneralHelp (line 60) | void PrintGeneralHelp(string executable)
FILE: source/OctodiffAsync/CommandLine/PatchCommand.cs
class PatchCommand (line 9) | [Command("patch", Description = "Given a basis file, and a delta, produc...
method PatchCommand (line 19) | public PatchCommand()
method GetHelp (line 29) | public void GetHelp(TextWriter writer)
method Execute (line 34) | public int Execute(string[] commandLineArguments)
FILE: source/OctodiffAsync/CommandLine/SignatureCommand.cs
class SignatureCommand (line 10) | [Command("signature", "sig", Description = "Given a basis file, creates ...
method SignatureCommand (line 18) | public SignatureCommand()
method GetHelp (line 27) | public void GetHelp(TextWriter writer)
method Execute (line 32) | public int Execute(string[] commandLineArguments)
FILE: source/OctodiffAsync/CommandLine/Support/CommandAttribute.cs
class CommandAttribute (line 5) | [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited...
method CommandAttribute (line 8) | public CommandAttribute(string name, params string[] aliases)
FILE: source/OctodiffAsync/CommandLine/Support/CommandException.cs
class CommandException (line 5) | class CommandException : Exception
method CommandException (line 7) | public CommandException(string message)
FILE: source/OctodiffAsync/CommandLine/Support/CommandLocator.cs
class CommandLocator (line 6) | class CommandLocator : ICommandLocator
method List (line 8) | public ICommandMetadata[] List()
method Find (line 18) | public ICommandMetadata Find(string name)
method Create (line 29) | public ICommand Create(ICommandMetadata metadata)
FILE: source/OctodiffAsync/CommandLine/Support/ICommand.cs
type ICommand (line 5) | interface ICommand
method GetHelp (line 7) | void GetHelp(TextWriter writer);
method Execute (line 8) | int Execute(string[] commandLineArguments);
FILE: source/OctodiffAsync/CommandLine/Support/ICommandLocator.cs
type ICommandLocator (line 3) | interface ICommandLocator
method List (line 5) | ICommandMetadata[] List();
method Find (line 6) | ICommandMetadata Find(string name);
method Create (line 7) | ICommand Create(ICommandMetadata metadata);
FILE: source/OctodiffAsync/CommandLine/Support/ICommandMetadata.cs
type ICommandMetadata (line 3) | interface ICommandMetadata
FILE: source/OctodiffAsync/CommandLine/Support/NDesk.Options.cs
class OptionValueCollection (line 152) | class OptionValueCollection : IList, IList<string>
method OptionValueCollection (line 158) | internal OptionValueCollection(OptionContext c)
method CopyTo (line 164) | void ICollection.CopyTo(Array array, int index) { (values as ICollecti...
method Add (line 170) | public void Add(string item) { values.Add(item); }
method Clear (line 171) | public void Clear() { values.Clear(); }
method Contains (line 172) | public bool Contains(string item) { return values.Contains(item); }
method CopyTo (line 173) | public void CopyTo(string[] array, int arrayIndex) { values.CopyTo(arr...
method Remove (line 174) | public bool Remove(string item) { return values.Remove(item); }
method GetEnumerator (line 180) | IEnumerator IEnumerable.GetEnumerator() { return values.GetEnumerator(...
method GetEnumerator (line 184) | public IEnumerator<string> GetEnumerator() { return values.GetEnumerat...
method Add (line 188) | int IList.Add(object value) { return (values as IList).Add(value); }
method Contains (line 189) | bool IList.Contains(object value) { return (values as IList).Contains(...
method IndexOf (line 190) | int IList.IndexOf(object value) { return (values as IList).IndexOf(val...
method Insert (line 191) | void IList.Insert(int index, object value) { (values as IList).Insert(...
method Remove (line 192) | void IList.Remove(object value) { (values as IList).Remove(value); }
method RemoveAt (line 193) | void IList.RemoveAt(int index) { (values as IList).RemoveAt(index); }
method IndexOf (line 199) | public int IndexOf(string item) { return values.IndexOf(item); }
method Insert (line 200) | public void Insert(int index, string item) { values.Insert(index, item...
method RemoveAt (line 201) | public void RemoveAt(int index) { values.RemoveAt(index); }
method AssertValid (line 203) | private void AssertValid(int index)
method ToList (line 230) | public List<string> ToList()
method ToArray (line 235) | public string[] ToArray()
method ToString (line 240) | public override string ToString()
class OptionContext (line 246) | class OptionContext
method OptionContext (line 254) | public OptionContext(OptionSet set)
type OptionValueType (line 289) | enum OptionValueType
class Option (line 296) | abstract class Option
method Option (line 304) | protected Option(string prototype, string description)
method Option (line 309) | protected Option(string prototype, string description, int maxValueCount)
method GetNames (line 346) | public string[] GetNames()
method GetValueSeparators (line 351) | public string[] GetValueSeparators()
method Parse (line 358) | protected static T Parse<T>(string value, OptionContext c)
method ParsePrototype (line 388) | private OptionValueType ParsePrototype()
method AddSeparators (line 431) | private static void AddSeparators(string name, int end, ICollection<st...
method Invoke (line 465) | public void Invoke(OptionContext c)
method OnParseComplete (line 473) | protected abstract void OnParseComplete(OptionContext c);
method ToString (line 475) | public override string ToString()
class OptionException (line 481) | [Serializable]
method OptionException (line 486) | public OptionException()
method OptionException (line 490) | public OptionException(string message, string optionName)
method OptionException (line 496) | public OptionException(string message, string optionName, Exception in...
method OptionException (line 502) | protected OptionException(SerializationInfo info, StreamingContext con...
method GetObjectData (line 513) | [SecurityPermission(SecurityAction.LinkDemand, SerializationFormatter ...
class OptionSet (line 523) | class OptionSet : KeyedCollection<string, Option>
method OptionSet (line 527) | public OptionSet()
method OptionSet (line 532) | public OptionSet(Converter<string, string> localizer)
method GetKeyForItem (line 544) | protected override string GetKeyForItem(Option item)
method GetOptionForName (line 555) | [Obsolete("Use KeyedCollection.this[string]")]
method InsertItem (line 570) | protected override void InsertItem(int index, Option item)
method RemoveItem (line 576) | protected override void RemoveItem(int index)
method SetItem (line 587) | protected override void SetItem(int index, Option item)
method AddImpl (line 594) | private void AddImpl(Option option)
method Add (line 616) | public new OptionSet Add(Option option)
method Positional (line 622) | public void Positional(string name, string description, Action<string>...
class ActionOption (line 627) | sealed class ActionOption : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
method Add (line 645) | public OptionSet Add(string prototype, Action<string> action)
method Add (line 650) | public OptionSet Add(string prototype, string description, Action<stri...
method Add (line 660) | public OptionSet Add(string prototype, OptionAction<string, string> ac...
method Add (line 665) | public OptionSet Add(string prototype, string description, OptionActio...
class ActionOption (line 675) | sealed class ActionOption<T> : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
class ActionOption (line 693) | sealed class ActionOption<TKey, TValue> : Option
method ActionOption (line 631) | public ActionOption(string prototype, string description, int count,...
method OnParseComplete (line 639) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 679) | public ActionOption(string prototype, string description, Action<T> ...
method OnParseComplete (line 687) | protected override void OnParseComplete(OptionContext c)
method ActionOption (line 697) | public ActionOption(string prototype, string description, OptionActi...
method OnParseComplete (line 705) | protected override void OnParseComplete(OptionContext c)
method Add (line 713) | public OptionSet Add<T>(string prototype, Action<T> action)
method Add (line 718) | public OptionSet Add<T>(string prototype, string description, Action<T...
method Add (line 723) | public OptionSet Add<TKey, TValue>(string prototype, OptionAction<TKey...
method Add (line 728) | public OptionSet Add<TKey, TValue>(string prototype, string descriptio...
method CreateOptionContext (line 733) | protected virtual OptionContext CreateOptionContext()
method Parse (line 739) | public List<string> Parse (IEnumerable<string> arguments)
method Parse (line 767) | public List<string> Parse(IEnumerable<string> arguments)
method Unprocessed (line 805) | private static bool Unprocessed(ICollection<string> extra, Option def,...
method GetOptionParts (line 821) | protected bool GetOptionParts(string argument, out string flag, out st...
method Parse (line 842) | protected virtual bool Parse(string argument, OptionContext c)
method ParseValue (line 883) | private void ParseValue(string option, OptionContext c)
method ParseBool (line 904) | private bool ParseBool(string option, string n, OptionContext c)
method ParseBundledValue (line 922) | private bool ParseBundledValue(string f, string n, OptionContext c)
method Invoke (line 960) | private static void Invoke(OptionContext c, string name, string value,...
method GetPositionals (line 970) | public List<string> GetPositionals()
method WriteOptionDescriptions (line 975) | public void WriteOptionDescriptions(TextWriter o)
method WriteOptionPrototype (line 1044) | bool WriteOptionPrototype(TextWriter o, Option p, ref int written)
method GetNextOptionIndex (line 1094) | static int GetNextOptionIndex(string[] names, int i)
method Write (line 1103) | static void Write(TextWriter o, ref int n, string s)
method GetArgumentName (line 1109) | private static string GetArgumentName(int index, int maxIndex, string ...
method GetDescription (line 1135) | private static string GetDescription(string description)
method GetLines (line 1182) | private static IEnumerable<string> GetLines(string description)
method IsEolChar (line 1208) | private static bool IsEolChar(char c)
method GetLineEnd (line 1213) | private static int GetLineEnd(int start, int length, string description)
FILE: source/OctodiffAsync/OctodiffAsyncProgram.cs
class OctodiffAsyncProgram (line 9) | public class OctodiffAsyncProgram
method Main (line 11) | static int Main(string[] args)
method WriteError (line 62) | static void WriteError(Exception ex, bool details = false)
method ExtractCommand (line 73) | private static string ExtractCommand(ICollection<string> args, out str...
Condensed preview — 121 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (407K chars).
[
{
"path": ".gitattributes",
"chars": 2518,
"preview": "###############################################################################\n# Set default behavior to automatically "
},
{
"path": ".gitignore",
"chars": 2597,
"preview": "## Ignore Visual Studio temporary files, build results, and\n## files generated by popular Visual Studio add-ons.\n\n# User"
},
{
"path": "LICENSE",
"chars": 11323,
"preview": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licens"
},
{
"path": "README.md",
"chars": 5858,
"preview": "# FastRsyncNet - C# delta syncing library\n\nThe Fast Rsync .NET library is Rsync implementation derived from [Octodiff](h"
},
{
"path": "source/.editorconfig",
"chars": 385,
"preview": "; EditorConfig to support per-solution formatting.\n; Use the EditorConfig VS add-in to make this work.\n; http://editorco"
},
{
"path": "source/FastRsync/Core/AggregateCopyOperationsDecorator.cs",
"chars": 2126,
"preview": "using System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Delta;\n\nnamespace FastRsync.Core\n"
},
{
"path": "source/FastRsync/Core/BinaryFormat.cs",
"chars": 984,
"preview": "using System.Text;\n\nnamespace FastRsync.Core\n{\n internal class BinaryFormat\n {\n public const int SignatureF"
},
{
"path": "source/FastRsync/Core/ChunkSignature.cs",
"chars": 596,
"preview": "using System;\n\nnamespace FastRsync.Core\n{\n public class ChunkSignature\n {\n public long StartOffset; "
},
{
"path": "source/FastRsync/Core/ChunkSignatureChecksumComparer.cs",
"chars": 395,
"preview": "using System.Collections.Generic;\n\nnamespace FastRsync.Core\n{\n class ChunkSignatureChecksumComparer : IComparer<Chun"
},
{
"path": "source/FastRsync/Core/JsonContext.cs",
"chars": 675,
"preview": "using FastRsync.Delta;\nusing FastRsync.Signature;\n\nnamespace FastRsync.Core\n{\n#if NET7_0_OR_GREATER\n using System.Te"
},
{
"path": "source/FastRsync/Core/JsonSerializationSettings.cs",
"chars": 595,
"preview": "using System.Text.Json;\nusing System.Text.Json.Serialization;\n\nnamespace FastRsync.Core\n{\n public class JsonSerializ"
},
{
"path": "source/FastRsync/Core/SupportedAlgorithms.cs",
"chars": 2789,
"preview": "using System;\nusing System.IO.Hashing;\nusing System.Security.Cryptography;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Co"
},
{
"path": "source/FastRsync/Delta/BinaryDeltaReader.cs",
"chars": 7705,
"preview": "using System;\nusing System.Collections;\nusing System.IO;\nusing System.Text.Json;\nusing System.Threading;\nusing System.Th"
},
{
"path": "source/FastRsync/Delta/BinaryDeltaWriter.cs",
"chars": 3230,
"preview": "using System;\nusing System.IO;\nusing System.Text.Json;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastR"
},
{
"path": "source/FastRsync/Delta/DataRange.cs",
"chars": 275,
"preview": "namespace FastRsync.Delta\n{\n public struct DataRange\n {\n public DataRange(long startOffset, long length)\n "
},
{
"path": "source/FastRsync/Delta/DeltaApplier.cs",
"chars": 4676,
"preview": "using System;\nusing System.Collections;\nusing System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing Fas"
},
{
"path": "source/FastRsync/Delta/DeltaBuilder.cs",
"chars": 12257,
"preview": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Threading;\nusin"
},
{
"path": "source/FastRsync/Delta/DeltaMetadata.cs",
"chars": 346,
"preview": "namespace FastRsync.Delta\n{\n public class DeltaMetadata\n {\n public string HashAlgorithm { get; set; }\n "
},
{
"path": "source/FastRsync/Delta/IDeltaReader.cs",
"chars": 630,
"preview": "using System;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Hash;\nusing FastRsync.Signature;\n\nna"
},
{
"path": "source/FastRsync/Delta/IDeltaWriter.cs",
"chars": 481,
"preview": "using System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Delta"
},
{
"path": "source/FastRsync/Diagnostics/ConsoleProgressReporter.cs",
"chars": 796,
"preview": "using System;\n\nnamespace FastRsync.Diagnostics\n{\n public class ConsoleProgressReporter : IProgress<ProgressReport>\n "
},
{
"path": "source/FastRsync/Diagnostics/ProgressReport.cs",
"chars": 464,
"preview": "namespace FastRsync.Diagnostics\n{\n public enum ProgressOperationType\n {\n BuildingSignatures,\n Hashi"
},
{
"path": "source/FastRsync/FastRsync.csproj",
"chars": 2080,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup Condition=\"'$(TargetFramework)' == 'net7.0'\">\n <LangVersion>12.0<"
},
{
"path": "source/FastRsync/Hash/Adler32RollingChecksum.cs",
"chars": 894,
"preview": "using System;\n\nnamespace FastRsync.Hash\n{\n public class Adler32RollingChecksum : IRollingChecksum\n {\n publi"
},
{
"path": "source/FastRsync/Hash/Adler32RollingChecksumV2.cs",
"chars": 1083,
"preview": "using System;\n\nnamespace FastRsync.Hash\n{\n [Obsolete(\"Adler32V2 has buggy mod operation implemented. See https://git"
},
{
"path": "source/FastRsync/Hash/Adler32RollingChecksumV3.cs",
"chars": 1171,
"preview": "using System.Reflection;\nusing System.Runtime.CompilerServices;\n\nnamespace FastRsync.Hash\n{\n public class Adler32Rol"
},
{
"path": "source/FastRsync/Hash/CryptographyHashAlgorithmWrapper.cs",
"chars": 1204,
"preview": "using System.IO;\nusing System.Security.Cryptography;\nusing System.Threading;\nusing System.Threading.Tasks;\n\nnamespace Fa"
},
{
"path": "source/FastRsync/Hash/IHashAlgorithm.cs",
"chars": 427,
"preview": "using System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\n\nnamespace FastRsync.Hash\n{\n public interface "
},
{
"path": "source/FastRsync/Hash/IRollingChecksum.cs",
"chars": 263,
"preview": "using System;\n\nnamespace FastRsync.Hash\n{\n public interface IRollingChecksum\n {\n string Name { get; }\n "
},
{
"path": "source/FastRsync/Hash/NonCryptographicHashAlgorithmWrapper.cs",
"chars": 1438,
"preview": "using System;\nusing System.IO.Hashing;\nusing System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\n\nnamespac"
},
{
"path": "source/FastRsync/Signature/ISignatureReader.cs",
"chars": 162,
"preview": "namespace FastRsync.Signature\n{\n public interface ISignatureReader\n {\n Signature ReadSignature();\n "
},
{
"path": "source/FastRsync/Signature/ISignatureWriter.cs",
"chars": 458,
"preview": "using System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\n\nnamespace FastRsync.Signature\n{\n public "
},
{
"path": "source/FastRsync/Signature/Signature.cs",
"chars": 1243,
"preview": "using System.Collections.Generic;\nusing FastRsync.Core;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Signature\n{\n publi"
},
{
"path": "source/FastRsync/Signature/SignatureBuilder.cs",
"chars": 7749,
"preview": "using System;\nusing System.IO;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\nusing FastRsy"
},
{
"path": "source/FastRsync/Signature/SignatureReader.cs",
"chars": 4956,
"preview": "using System;\nusing System.Collections;\nusing System.IO;\nusing System.Text.Json;\nusing FastRsync.Core;\nusing FastRsync."
},
{
"path": "source/FastRsync/Signature/SignatureWriter.cs",
"chars": 2288,
"preview": "using System.IO;\nusing System.Text.Json;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\n\nn"
},
{
"path": "source/FastRsync.BackwardCompatibilityTests/BackwardCompatibilityTests.cs",
"chars": 6245,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing FastRsy"
},
{
"path": "source/FastRsync.BackwardCompatibilityTests/FastRsync.BackwardCompatibilityTests.csproj",
"chars": 1178,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n\n <PropertyGroup>\n <TargetFramework>net472</TargetFramework>\n <LangVersion>7.3"
},
{
"path": "source/FastRsync.Benchmarks/BuildPatchBenchmark.cs",
"chars": 4654,
"preview": "using System;\nusing System.IO;\nusing BenchmarkDotNet.Attributes;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing Fas"
},
{
"path": "source/FastRsync.Benchmarks/FastRsync.Benchmarks.csproj",
"chars": 829,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n\n <PropertyGroup>\n <OutputType>Exe</OutputType>\n <TargetFramework>net10.0</Tar"
},
{
"path": "source/FastRsync.Benchmarks/HashBenchmark.cs",
"chars": 3512,
"preview": "using System;\nusing System.Data.HashFunction.xxHash;\nusing System.IO;\nusing System.IO.Hashing;\nusing System.Threading.T"
},
{
"path": "source/FastRsync.Benchmarks/Program.cs",
"chars": 277,
"preview": "using System.Reflection;\nusing BenchmarkDotNet.Running;\n\nnamespace FastRsync.Benchmarks\n{\n class Program\n {\n "
},
{
"path": "source/FastRsync.Benchmarks/RollingChecksumBenchmark.cs",
"chars": 2266,
"preview": "using System;\nusing BenchmarkDotNet.Attributes;\nusing FastRsync.Core;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Benchm"
},
{
"path": "source/FastRsync.Benchmarks/SignatureBenchmark.cs",
"chars": 4014,
"preview": "using System;\nusing System.IO;\nusing BenchmarkDotNet.Attributes;\nusing FastRsync.Core;\nusing FastRsync.Signature;\nusing"
},
{
"path": "source/FastRsync.Compression/FastRsync.Compression.csproj",
"chars": 989,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n\n <PropertyGroup>\n <TargetFramework>netstandard2.0</TargetFramework>\n <Generat"
},
{
"path": "source/FastRsync.Compression/GZip.cs",
"chars": 4238,
"preview": "// this implementation is influenced by pigz tool by Mark Adler: https://github.com/madler/pigz/blob/master/pigz.c\n// p"
},
{
"path": "source/FastRsync.Tests/Adler32RollingChecksumAlgorithmsTests.cs",
"chars": 5020,
"preview": "using FastRsync.Hash;\nusing NSubstitute;\nusing NUnit.Framework;\nusing NUnit.Framework.Legacy;\nusing System;\nusing Syste"
},
{
"path": "source/FastRsync.Tests/Adler32RollingChecksumTests.cs",
"chars": 1924,
"preview": "using FastRsync.Hash;\nusing NUnit.Framework;\n\nnamespace FastRsync.Tests;\n\n[TestFixture]\npublic class Adler32RollingChec"
},
{
"path": "source/FastRsync.Tests/Adler32RollingChecksumV2Tests.cs",
"chars": 1847,
"preview": "using FastRsync.Hash;\nusing NUnit.Framework;\n\nnamespace FastRsync.Tests;\n\n[TestFixture]\npublic class Adler32RollingChec"
},
{
"path": "source/FastRsync.Tests/Adler32RollingChecksumV3Tests.cs",
"chars": 2436,
"preview": "using FastRsync.Hash;\nusing NUnit.Framework;\nusing System;\n\nnamespace FastRsync.Tests;\n\n[TestFixture]\npublic class Adle"
},
{
"path": "source/FastRsync.Tests/BackwardCompatibilityTests.cs",
"chars": 3116,
"preview": "using NUnit.Framework;\nusing System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\nusing FastRsy"
},
{
"path": "source/FastRsync.Tests/CommonAsserts.cs",
"chars": 1106,
"preview": "using System.IO;\nusing FastRsync.Hash;\nusing FastRsync.Signature;\nusing NUnit.Framework;\n\nnamespace FastRsync.Tests;\n\nc"
},
{
"path": "source/FastRsync.Tests/DeltaReaderTests.cs",
"chars": 3581,
"preview": "using System.IO;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing FastRsync.Signature;\nusing FastRsync.Tests.FastRsyn"
},
{
"path": "source/FastRsync.Tests/FastRsync.Tests.csproj",
"chars": 2467,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup>\n <TargetFramework>net10.0</TargetFramework>\n </PropertyGroup>\n "
},
{
"path": "source/FastRsync.Tests/FastRsyncLegacy/BinaryDeltaReaderLegacy.cs",
"chars": 7704,
"preview": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\nusing Sys"
},
{
"path": "source/FastRsync.Tests/FastRsyncLegacy/DeltaMetadataLegacy.cs",
"chars": 236,
"preview": "namespace FastRsync.Tests.FastRsyncLegacy;\n\ninternal class DeltaMetadataLegacy\n{\n public string HashAlgorithm { get;"
},
{
"path": "source/FastRsync.Tests/FastRsyncLegacy/IDeltaReaderLegacy.cs",
"chars": 523,
"preview": "using System;\nusing System.Threading.Tasks;\nusing FastRsync.Hash;\nusing FastRsync.Signature;\n\nnamespace FastRsync.Tests"
},
{
"path": "source/FastRsync.Tests/FastRsyncLegacy/NewtonsoftJsonSerializationSettings.cs",
"chars": 530,
"preview": "using Newtonsoft.Json.Serialization;\nusing Newtonsoft.Json;\n\nnamespace FastRsync.Tests.FastRsyncLegacy;\n\npublic class N"
},
{
"path": "source/FastRsync.Tests/GZipTests.cs",
"chars": 8426,
"preview": "using System;\nusing System.IO;\nusing System.IO.Compression;\nusing System.Threading;\nusing System.Threading.Tasks;\nusing"
},
{
"path": "source/FastRsync.Tests/HashTests.cs",
"chars": 3457,
"preview": "using NUnit.Framework;\nusing System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing Azure.Storage.Blobs;\nusing Az"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/IOctodiffDeltaWriter.cs",
"chars": 393,
"preview": "using System.IO;\nusing System.Threading.Tasks;\nusing FastRsync.Delta;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests."
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/IOctodiffSignatureWriter.cs",
"chars": 1080,
"preview": "using System.IO;\nusing FastRsync.Core;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests.OctodiffLegacy;\n\npublic interfa"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffAggregateCopyOperationsDecorator.cs",
"chars": 1699,
"preview": "using System.IO;\nusing FastRsync.Delta;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests.OctodiffLegacy;\n\n// This decor"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryDeltaWriter.cs",
"chars": 1868,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Delta;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests.OctodiffLegacy;\n"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffBinaryFormat.cs",
"chars": 473,
"preview": "using System.Text;\n\nnamespace FastRsync.Tests.OctodiffLegacy;\n\nclass OctodiffBinaryFormat\n{\n public static readonly "
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffDeltaBuilder.cs",
"chars": 5225,
"preview": "using System.Collections;\nusing System.Collections.Generic;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Delt"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffSignature.cs",
"chars": 615,
"preview": "using System.Collections.Generic;\nusing FastRsync.Core;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests.OctodiffLegacy"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureBuilder.cs",
"chars": 2446,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Hash;\n\nnamespace FastRsync.Tests.OctodiffLegacy;\n\n"
},
{
"path": "source/FastRsync.Tests/OctodiffLegacy/OctodiffSignatureReader.cs",
"chars": 2677,
"preview": "using System;\nusing System.Collections;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Diagnostics;\n\nnamespace "
},
{
"path": "source/FastRsync.Tests/PatchingAsyncTests.cs",
"chars": 11229,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing FastRsy"
},
{
"path": "source/FastRsync.Tests/PatchingBigFilesTests.cs",
"chars": 4260,
"preview": "using FastRsync.Core;\nusing FastRsync.Delta;\nusing FastRsync.Diagnostics;\nusing FastRsync.Signature;\nusing NSubstitute;"
},
{
"path": "source/FastRsync.Tests/PatchingSyncTests.cs",
"chars": 6363,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing FastRsync.Diagnostics;\nusing FastRsyn"
},
{
"path": "source/FastRsync.Tests/SignatureBuilderAsyncRandomDataTests.cs",
"chars": 4768,
"preview": "using System;\nusing System.IO;\nusing System.Security.Cryptography;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\n"
},
{
"path": "source/FastRsync.Tests/SignatureBuilderSyncRandomDataTests.cs",
"chars": 4605,
"preview": "using System;\nusing System.IO;\nusing System.Security.Cryptography;\nusing FastRsync.Core;\nusing FastRsync.Diagnostics;\nu"
},
{
"path": "source/FastRsync.Tests/SignatureBuilderTests.cs",
"chars": 4818,
"preview": "using System;\nusing System.IO;\nusing System.Threading.Tasks;\nusing FastRsync.Core;\nusing FastRsync.Diagnostics;\nusing F"
},
{
"path": "source/FastRsync.Tests/SignatureReaderTests.cs",
"chars": 1959,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Diagnostics;\nusing FastRsync.Hash;\nusing FastRsync"
},
{
"path": "source/FastRsync.Tests/Utils.cs",
"chars": 1220,
"preview": "using System;\nusing System.IO;\nusing System.Security.Cryptography;\nusing FastRsync.Signature;\n\nnamespace FastRsync.Test"
},
{
"path": "source/FastRsync.sln",
"chars": 4623,
"preview": "\nMicrosoft Visual Studio Solution File, Format Version 12.00\n# Visual Studio Version 17\nVisualStudioVersion = 17.8.3430"
},
{
"path": "source/Octodiff/CommandLine/DeltaCommand.cs",
"chars": 3531,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing Fas"
},
{
"path": "source/Octodiff/CommandLine/ExplainDeltaCommand.cs",
"chars": 2118,
"preview": "using System;\nusing System.IO;\nusing System.Linq;\nusing FastRsync.Delta;\nusing Octodiff.CommandLine.Support;\n\nnamespace "
},
{
"path": "source/Octodiff/CommandLine/HelpCommand.cs",
"chars": 3046,
"preview": "using System;\nusing System.IO;\nusing System.Linq;\nusing Octodiff.CommandLine.Support;\n\nnamespace Octodiff.CommandLine\n{\n"
},
{
"path": "source/Octodiff/CommandLine/PatchCommand.cs",
"chars": 3308,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Delta;\nusing FastRsync.Diagnostics;\nusing Octodiff.CommandLine.Support;\n"
},
{
"path": "source/Octodiff/CommandLine/SignatureCommand.cs",
"chars": 3101,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing FastRsync.Diagnostics;\nusing FastRsync.Signature"
},
{
"path": "source/Octodiff/CommandLine/Support/CommandAttribute.cs",
"chars": 537,
"preview": "using System;\n\nnamespace Octodiff.CommandLine.Support\n{\n [AttributeUsage(AttributeTargets.Class, AllowMultiple = fals"
},
{
"path": "source/Octodiff/CommandLine/Support/CommandException.cs",
"chars": 204,
"preview": "using System;\n\nnamespace Octodiff.CommandLine.Support\n{\n class CommandException : Exception\n {\n public Comm"
},
{
"path": "source/Octodiff/CommandLine/Support/CommandLocator.cs",
"chars": 1807,
"preview": "using System;\nusing System.Linq;\n\nnamespace Octodiff.CommandLine.Support\n{\n class CommandLocator : ICommandLocator\n "
},
{
"path": "source/Octodiff/CommandLine/Support/ICommand.cs",
"chars": 190,
"preview": "using System.IO;\n\nnamespace Octodiff.CommandLine.Support\n{\n interface ICommand\n {\n void GetHelp(TextWriter"
},
{
"path": "source/Octodiff/CommandLine/Support/ICommandLocator.cs",
"chars": 215,
"preview": "namespace Octodiff.CommandLine.Support\n{\n interface ICommandLocator\n {\n ICommandMetadata[] List();\n "
},
{
"path": "source/Octodiff/CommandLine/Support/ICommandMetadata.cs",
"chars": 219,
"preview": "namespace Octodiff.CommandLine.Support\n{\n interface ICommandMetadata\n {\n string Name { get; }\n strin"
},
{
"path": "source/Octodiff/CommandLine/Support/NDesk.Options.cs",
"chars": 42424,
"preview": "//\n// Options.cs\n//\n// Authors:\n// Jonathan Pryor <jpryor@novell.com>\n//\n// Copyright (C) 2008 Novell (http://www.nove"
},
{
"path": "source/Octodiff/Octodiff.csproj",
"chars": 2152,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup>\n <TargetFramework>net472</TargetFramework>\n <LangVersion>7.3<"
},
{
"path": "source/Octodiff/OctodiffProgram.cs",
"chars": 2313,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\nusing Octodiff.CommandLine;\nusing O"
},
{
"path": "source/Octodiff/Properties/AssemblyInfo.cs",
"chars": 1350,
"preview": "using System.Reflection;\nusing System.Runtime.InteropServices;\n\n// General Information about an assembly is controlled "
},
{
"path": "source/Octodiff/app.config",
"chars": 158,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<configuration>\n<startup><supportedRuntime version=\"v4.0\" sku=\".NETFramework,Vers"
},
{
"path": "source/Octodiff.Tests/DeltaFixture.cs",
"chars": 3608,
"preview": "using System;\nusing System.Diagnostics;\nusing System.IO;\nusing System.Text.RegularExpressions;\nusing NUnit.Framework;\nu"
},
{
"path": "source/Octodiff.Tests/HelpFixture.cs",
"chars": 2039,
"preview": "using NUnit.Framework;\nusing Octodiff.Tests.Util;\n\nnamespace Octodiff.Tests\n{\n [TestFixture]\n public class HelpFi"
},
{
"path": "source/Octodiff.Tests/Octodiff.Tests.csproj",
"chars": 2345,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup>\n <TargetFramework>net472</TargetFramework>\n <LangVersion>7.3<"
},
{
"path": "source/Octodiff.Tests/PackageGenerator.cs",
"chars": 2297,
"preview": "using System;\nusing System.IO;\nusing System.IO.Packaging;\nusing System.Linq;\nusing NUnit.Framework;\n\nnamespace Octodiff"
},
{
"path": "source/Octodiff.Tests/PatchFixture.cs",
"chars": 4713,
"preview": "using System;\nusing System.IO;\nusing System.Security.Cryptography;\nusing NUnit.Framework;\nusing Octodiff.Tests.Util;\n\nna"
},
{
"path": "source/Octodiff.Tests/Properties/AssemblyInfo.cs",
"chars": 1362,
"preview": "using System.Reflection;\nusing System.Runtime.InteropServices;\n\n// General Information about an assembly is controlled "
},
{
"path": "source/Octodiff.Tests/SignatureFixture.cs",
"chars": 3737,
"preview": "using System.Diagnostics;\nusing System.IO;\nusing NUnit.Framework;\nusing Octodiff.Tests.Util;\n\nnamespace Octodiff.Tests\n"
},
{
"path": "source/Octodiff.Tests/Timings.cs",
"chars": 1915,
"preview": "using System;\nusing System.Diagnostics;\nusing System.IO;\nusing NUnit.Framework;\nusing Octodiff.Tests.Util;\n\nnamespace Oc"
},
{
"path": "source/Octodiff.Tests/Util/CommandLineFixture.cs",
"chars": 1728,
"preview": "using System;\nusing System.Diagnostics;\nusing System.Text;\nusing NUnit.Framework;\nusing OctodiffAsync;\nusing Octopus.Pl"
},
{
"path": "source/Octodiff.Tests/Util/SilentProcessRunner.cs",
"chars": 4378,
"preview": "using System;\nusing System.Diagnostics;\nusing System.Runtime.InteropServices;\nusing System.Text;\nusing System.Threading"
},
{
"path": "source/OctodiffAsync/CommandLine/DeltaCommand.cs",
"chars": 3571,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing FastRsync.Core;\nusing FastRsync.Delta;\nusing Fas"
},
{
"path": "source/OctodiffAsync/CommandLine/ExplainDeltaCommand.cs",
"chars": 2360,
"preview": "using System;\nusing System.IO;\nusing System.Linq;\nusing System.Threading.Tasks;\nusing FastRsync.Delta;\nusing OctodiffAsy"
},
{
"path": "source/OctodiffAsync/CommandLine/HelpCommand.cs",
"chars": 3056,
"preview": "using System;\nusing System.IO;\nusing System.Linq;\nusing OctodiffAsync.CommandLine.Support;\n\nnamespace OctodiffAsync.Comm"
},
{
"path": "source/OctodiffAsync/CommandLine/PatchCommand.cs",
"chars": 3348,
"preview": "using System;\nusing System.IO;\nusing FastRsync.Delta;\nusing FastRsync.Diagnostics;\nusing OctodiffAsync.CommandLine.Supp"
},
{
"path": "source/OctodiffAsync/CommandLine/SignatureCommand.cs",
"chars": 3141,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing FastRsync.Diagnostics;\nusing FastRsync.Signature"
},
{
"path": "source/OctodiffAsync/CommandLine/Support/CommandAttribute.cs",
"chars": 542,
"preview": "using System;\n\nnamespace OctodiffAsync.CommandLine.Support\n{\n [AttributeUsage(AttributeTargets.Class, AllowMultiple ="
},
{
"path": "source/OctodiffAsync/CommandLine/Support/CommandException.cs",
"chars": 209,
"preview": "using System;\n\nnamespace OctodiffAsync.CommandLine.Support\n{\n class CommandException : Exception\n {\n public"
},
{
"path": "source/OctodiffAsync/CommandLine/Support/CommandLocator.cs",
"chars": 1812,
"preview": "using System;\nusing System.Linq;\n\nnamespace OctodiffAsync.CommandLine.Support\n{\n class CommandLocator : ICommandLocat"
},
{
"path": "source/OctodiffAsync/CommandLine/Support/ICommand.cs",
"chars": 195,
"preview": "using System.IO;\n\nnamespace OctodiffAsync.CommandLine.Support\n{\n interface ICommand\n {\n void GetHelp(TextW"
},
{
"path": "source/OctodiffAsync/CommandLine/Support/ICommandLocator.cs",
"chars": 220,
"preview": "namespace OctodiffAsync.CommandLine.Support\n{\n interface ICommandLocator\n {\n ICommandMetadata[] List();\n "
},
{
"path": "source/OctodiffAsync/CommandLine/Support/ICommandMetadata.cs",
"chars": 224,
"preview": "namespace OctodiffAsync.CommandLine.Support\n{\n interface ICommandMetadata\n {\n string Name { get; }\n "
},
{
"path": "source/OctodiffAsync/CommandLine/Support/NDesk.Options.cs",
"chars": 42437,
"preview": "//\n// Options.cs\n//\n// Authors:\n// Jonathan Pryor <jpryor@novell.com>\n//\n// Copyright (C) 2008 Novell (http://www.nove"
},
{
"path": "source/OctodiffAsync/OctodiffAsync.csproj",
"chars": 2162,
"preview": "<Project Sdk=\"Microsoft.NET.Sdk\">\n <PropertyGroup>\n <TargetFramework>net472</TargetFramework>\n <LangVersion>7.3<"
},
{
"path": "source/OctodiffAsync/OctodiffAsyncProgram.cs",
"chars": 2300,
"preview": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\nusing OctodiffAsync.CommandLine.Sup"
},
{
"path": "source/OctodiffAsync/Properties/AssemblyInfo.cs",
"chars": 1360,
"preview": "using System.Reflection;\nusing System.Runtime.InteropServices;\n\n// General Information about an assembly is controlled "
},
{
"path": "source/OctodiffAsync/app.config",
"chars": 158,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<configuration>\n<startup><supportedRuntime version=\"v4.0\" sku=\".NETFramework,Vers"
},
{
"path": "source/Tests.ps1",
"chars": 5026,
"preview": "\nfunction Assert-That ($val, $error) {\n if ($val -ne $true) {\n Write-Error \"Test failed: $error\"\n }\n}\n\nfunc"
}
]
About this extraction
This page contains the full source code of the GrzegorzBlok/FastRsyncNet GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 121 files (375.9 KB), approximately 86.7k tokens, and a symbol index with 611 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.