[
{
"path": ".editorconfig",
"content": "# top-most EditorConfig file\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\nindent_style = space\nindent_size = 2\ninsert_final_newline = true\ntrim_trailing_whitespace = true\n\n# Visual Studio Spell checker configs (https://learn.microsoft.com/en-us/visualstudio/ide/text-spell-checker?view=vs-2022#how-to-customize-the-spell-checker)\nspelling_exclusion_path = ./exclusion.dic\n\n[*.cs]\nindent_size = 4\ncharset = utf-8-bom\nend_of_line = unset\n\n# Solution files\n[*.{sln,slnx}]\nend_of_line = unset\n\n# MSBuild project files\n[*.{csproj,props,targets}]\nend_of_line = unset\n\n# Xml config files\n[*.{ruleset,config,nuspec,resx,runsettings,DotSettings}]\nend_of_line = unset\n\n[*{_AssemblyInfo.cs,.notsupported.cs}]\ngenerated_code = true\n\n# C# code style settings\n[*.{cs}]\ndotnet_style_coalesce_expression = true:suggestion\ndotnet_style_null_propagation = true:suggestion\ndotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion\ndotnet_style_prefer_auto_properties = true:suggestion\ndotnet_style_object_initializer = true:suggestion\ndotnet_style_prefer_collection_expression = true:suggestion\ndotnet_style_collection_initializer = true:suggestion\ndotnet_style_prefer_simplified_boolean_expressions = true:suggestion\ndotnet_style_prefer_conditional_expression_over_assignment = true:silent\ndotnet_style_prefer_conditional_expression_over_return = true:silent\ndotnet_style_explicit_tuple_names = true:suggestion\ndotnet_style_prefer_inferred_tuple_names = true:suggestion\ndotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion\ndotnet_style_prefer_compound_assignment = true:suggestion\ndotnet_style_prefer_simplified_interpolation = true:suggestion\ndotnet_style_namespace_match_folder = true:suggestion\ndotnet_style_readonly_field = true:suggestion\ndotnet_style_predefined_type_for_member_access = true:suggestion\ndotnet_style_predefined_type_for_locals_parameters_members = true:suggestion\ndotnet_style_require_accessibility_modifiers = for_non_interface_members:silent\ndotnet_style_allow_statement_immediately_after_block_experimental = true:silent\ndotnet_style_allow_multiple_blank_lines_experimental = true:silent\ndotnet_code_quality_unused_parameters = non_public:suggestion\ndotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent\ndotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent\ndotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent\ndotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent\ndotnet_style_qualification_for_method = false:none\ndotnet_style_qualification_for_property = false:none\ndotnet_style_qualification_for_field = false:none\ndotnet_style_qualification_for_event = false:none\n\n# New line preferences\ncsharp_new_line_before_open_brace = all\ncsharp_new_line_before_else = true\ncsharp_new_line_before_catch = true\ncsharp_new_line_before_finally = true\ncsharp_new_line_before_members_in_object_initializers = true\ncsharp_new_line_before_members_in_anonymous_types = true\ncsharp_new_line_between_query_expression_clauses = true\n\n# Indentation preferences\ncsharp_indent_block_contents = true\ncsharp_indent_braces = false\ncsharp_indent_case_contents = true\ncsharp_indent_case_contents_when_block = true\ncsharp_indent_switch_labels = true\ncsharp_indent_labels = one_less_than_current\n\n# Modifier preferences\ncsharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion\n\n# avoid this. unless absolutely necessary\ndotnet_style_qualification_for_field = false:none\ndotnet_style_qualification_for_property = false:none\ndotnet_style_qualification_for_method = false:none\ndotnet_style_qualification_for_event = false:none\n\n# Types: use keywords instead of BCL types, and permit var only when the type is clear\ncsharp_style_var_for_built_in_types = false:none\ncsharp_style_var_when_type_is_apparent = false:none\ncsharp_style_var_elsewhere = false:none\ndotnet_style_predefined_type_for_locals_parameters_members = true:suggestion\ndotnet_style_predefined_type_for_member_access = true:suggestion\n\n# name all constant fields using PascalCase\ndotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion\ndotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields\ndotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style\ndotnet_naming_symbols.constant_fields.applicable_kinds = field\ndotnet_naming_symbols.constant_fields.required_modifiers = const\ndotnet_naming_style.pascal_case_style.capitalization = pascal_case\n\n# static fields\ndotnet_naming_rule.static_fields_should_have_prefix.severity = none\ndotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields\ndotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style\ndotnet_naming_symbols.static_fields.applicable_kinds = field\ndotnet_naming_symbols.static_fields.required_modifiers = static\ndotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected\ndotnet_naming_style.static_prefix_style.required_prefix = s_\ndotnet_naming_style.static_prefix_style.capitalization = camel_case\n\n# internal and private fields\ndotnet_naming_rule.camel_case_for_private_internal_fields.severity = none\ndotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields\ndotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style\ndotnet_naming_symbols.private_internal_fields.applicable_kinds = field\ndotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal\ndotnet_naming_style.camel_case_underscore_style.required_prefix = _\ndotnet_naming_style.camel_case_underscore_style.capitalization = camel_case\n\n# Code style defaults\ncsharp_using_directive_placement = outside_namespace:suggestion\ncsharp_prefer_braces = true:silent\ncsharp_preserve_single_line_blocks = true:none\ncsharp_preserve_single_line_statements = false:none\ncsharp_prefer_static_local_function = true:suggestion\ncsharp_prefer_simple_using_statement = false:none\ncsharp_style_prefer_switch_expression = true:suggestion\n\n# Code quality\ndotnet_style_readonly_field = true:suggestion\ndotnet_code_quality_unused_parameters = non_public:suggestion\n\n# Expression-level preferences\ndotnet_style_object_initializer = true:suggestion\ndotnet_style_collection_initializer = true:suggestion\ndotnet_style_explicit_tuple_names = true:suggestion\ndotnet_style_coalesce_expression = true:suggestion\ndotnet_style_null_propagation = true:suggestion\ndotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion\ndotnet_style_prefer_inferred_tuple_names = true:suggestion\ndotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion\ndotnet_style_prefer_auto_properties = true:suggestion\ndotnet_style_prefer_conditional_expression_over_assignment = true:silent\ndotnet_style_prefer_conditional_expression_over_return = true:silent\ncsharp_prefer_simple_default_expression = true:suggestion\n\n# Expression-bodied members\ncsharp_style_expression_bodied_methods = true:silent\ncsharp_style_expression_bodied_constructors = true:silent\ncsharp_style_expression_bodied_operators = true:silent\ncsharp_style_expression_bodied_properties = true:silent\ncsharp_style_expression_bodied_indexers = true:silent\ncsharp_style_expression_bodied_accessors = true:silent\ncsharp_style_expression_bodied_lambdas = true:silent\ncsharp_style_expression_bodied_local_functions = true:silent\n\n# Pattern matching\ncsharp_style_pattern_matching_over_is_with_cast_check = true:suggestion\ncsharp_style_pattern_matching_over_as_with_null_check = true:suggestion\ncsharp_style_inlined_variable_declaration = true:suggestion\n\n# Null checking preferences\ncsharp_style_throw_expression = true:suggestion\ncsharp_style_conditional_delegate_call = true:suggestion\n\n# Other features\ncsharp_style_prefer_index_operator = false:none\ncsharp_style_prefer_range_operator = false:none\ncsharp_style_pattern_local_over_anonymous_function = false:none\n\n# Space preferences\ncsharp_space_after_cast = false\ncsharp_space_after_colon_in_inheritance_clause = true\ncsharp_space_after_comma = true\ncsharp_space_after_dot = false\ncsharp_space_after_keywords_in_control_flow_statements = true\ncsharp_space_after_semicolon_in_for_statement = true\ncsharp_space_around_binary_operators = before_and_after\ncsharp_space_around_declaration_statements = do_not_ignore\ncsharp_space_before_colon_in_inheritance_clause = true\ncsharp_space_before_comma = false\ncsharp_space_before_dot = false\ncsharp_space_before_open_square_brackets = false\ncsharp_space_before_semicolon_in_for_statement = false\ncsharp_space_between_empty_square_brackets = false\ncsharp_space_between_method_call_empty_parameter_list_parentheses = false\ncsharp_space_between_method_call_name_and_opening_parenthesis = false\ncsharp_space_between_method_call_parameter_list_parentheses = false\ncsharp_space_between_method_declaration_empty_parameter_list_parentheses = false\ncsharp_space_between_method_declaration_name_and_open_parenthesis = false\ncsharp_space_between_method_declaration_parameter_list_parentheses = false\ncsharp_space_between_parentheses = false\ncsharp_space_between_square_brackets = false\n\n# Analyzers\ndotnet_code_quality.CA1052.api_surface = private, internal\ndotnet_code_quality.CA1802.api_surface = private, internal\ndotnet_code_quality.CA1822.api_surface = private, internal\ndotnet_code_quality.CA2208.api_surface = public\n\n# IDE0008: Use explicit type\ndotnet_diagnostic.IDE0008.severity = none\n\n# IDE0090: Use 'new(...)'\ndotnet_diagnostic.IDE0090.severity = none\n\n# IDE0040: Add accessibility modifiers\ndotnet_diagnostic.IDE0040.severity = none\n\n# Nullability in reference types of interface implemented by the base type doesn't match\ndotnet_diagnostic.CS8644.severity = none\n\ndotnet_diagnostic.CA1816.severity = none\n\ndotnet_diagnostic.IDE1006.severity = none\n\n#Remove unnecessary suppression\ndotnet_diagnostic.IDE0079.severity = none\n\ndotnet_diagnostic.IDE0130.severity = none\n\ndotnet_diagnostic.CA1822.severity = none\n\ncsharp_style_prefer_switch_expression = false:suggestion\n\ncsharp_style_pattern_matching_over_as_with_null_check = false:suggestion\n\ndotnet_naming_symbols.functional_symbols.applicable_kinds = property,method,event,delegate\ndotnet_naming_style.pascal_case_style.capitalization = pascal_case\ndotnet_naming_rule.functional_symbols_must_be_capitalized.symbols = functional_symbols\ndotnet_naming_rule.functional_symbols_must_be_capitalized.style = pascal_case_style\ndotnet_naming_rule.functional_symbols_must_be_capitalized.severity = warning\n\ndotnet_naming_symbols.public_symbols.applicable_kinds = property,method,field,event,delegate\ndotnet_naming_symbols.public_symbols.applicable_accessibilities = public\ndotnet_naming_symbols.public_symbols.required_modifiers = readonly\ndotnet_naming_style.first_word_upper_case_style.capitalization = first_word_upper\ndotnet_naming_rule.public_members_must_be_capitalized.symbols = public_symbols\ndotnet_naming_rule.public_members_must_be_capitalized.style = first_word_upper_case_style\ndotnet_naming_rule.public_members_must_be_capitalized.severity = warning\n\ncsharp_style_expression_bodied_methods = false:silent\ncsharp_style_expression_bodied_constructors = false:silent\ncsharp_style_expression_bodied_operators = false:silent\ncsharp_style_namespace_declarations = file_scoped:suggestion\ncsharp_style_prefer_method_group_conversion = true:silent\ncsharp_style_prefer_top_level_statements = true:silent\ncsharp_style_prefer_primary_constructors = true:suggestion\ncsharp_style_prefer_null_check_over_type_check = true:suggestion\ncsharp_style_prefer_local_over_anonymous_function = true:suggestion\ncsharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion\ncsharp_style_prefer_tuple_swap = true:suggestion\ncsharp_style_prefer_utf8_string_literals = true:suggestion\ncsharp_style_deconstructed_variable_declaration = true:suggestion\ncsharp_style_unused_value_assignment_preference = discard_variable:suggestion\ncsharp_style_unused_value_expression_statement_preference = discard_variable:silent\ncsharp_style_prefer_readonly_struct_member = true:suggestion\ncsharp_style_prefer_readonly_struct = true:suggestion\ncsharp_style_allow_embedded_statements_on_same_line_experimental = true:silent\ncsharp_style_allow_blank_line_after_token_in_arrow_expression_clause_experimental = true:silent\ncsharp_style_allow_blank_line_after_token_in_conditional_expression_experimental = true:silent\ncsharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true:silent\ncsharp_style_allow_blank_lines_between_consecutive_braces_experimental = true:silent\ncsharp_style_prefer_pattern_matching = true:silent\ncsharp_style_prefer_extended_property_pattern = true:suggestion\ncsharp_style_prefer_not_pattern = true:suggestion\n"
},
{
"path": ".github/dependabot.yaml",
"content": "# ref: https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot\nversion: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\" # Check for updates to GitHub Actions every week\n groups:\n dependencies:\n patterns:\n - \"*\"\n cooldown:\n default-days: 14 # Wait 14 days before creating another PR for the same dependency. This will prevent vulnerability on the package impact.\n ignore:\n # I just want update action when major/minor version is updated. patch updates are too noisy.\n - dependency-name: \"*\"\n update-types:\n - version-update:semver-patch\n"
},
{
"path": ".github/workflows/build-debug.yaml",
"content": "name: Build-Debug\n\non:\n push:\n branches:\n - \"main\"\n pull_request:\n branches:\n - \"main\"\n\njobs:\n build-dotnet:\n permissions:\n contents: read\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n steps:\n - uses: Cysharp/Actions/.github/actions/checkout@main\n - uses: Cysharp/Actions/.github/actions/setup-dotnet@main\n - run: dotnet build -c Release\n - run: dotnet test -c Release --no-build\n - run: dotnet pack -c Release --no-build -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -o $GITHUB_WORKSPACE/artifacts\n"
},
{
"path": ".github/workflows/build-release.yaml",
"content": "name: Build-Release\n\non:\n workflow_dispatch:\n inputs:\n tag:\n description: \"tag: git tag you want create. (sample 1.0.0)\"\n required: true\n dry-run:\n description: \"dry-run: true will never create relase/nuget.\"\n required: true\n default: false\n type: boolean\n\njobs:\n build-dotnet:\n permissions:\n contents: read\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n steps:\n - uses: Cysharp/Actions/.github/actions/checkout@main\n - uses: Cysharp/Actions/.github/actions/setup-dotnet@main\n - run: dotnet build -c Release -p:Version=${{ inputs.tag }}\n - run: dotnet test -c Release --no-build\n - run: dotnet pack -c Release --no-build -p:Version=${{ inputs.tag }} -o ./publish\n # Store artifacts.\n - uses: Cysharp/Actions/.github/actions/upload-artifact@main\n with:\n name: nuget\n path: ./publish/\n\n # release\n create-release:\n needs: [build-dotnet]\n permissions:\n contents: write\n id-token: write # required for NuGet Trusted Publish\n uses: Cysharp/Actions/.github/workflows/create-release.yaml@main\n with:\n commit-id: \"\"\n dry-run: ${{ inputs.dry-run }}\n tag: ${{ inputs.tag }}\n nuget-push: true\n release-upload: false\n secrets: inherit\n"
},
{
"path": ".github/workflows/stale.yaml",
"content": "name: \"Close stale issues\"\n\non:\n workflow_dispatch:\n schedule:\n - cron: \"0 0 * * *\"\n\njobs:\n stale:\n permissions:\n contents: read\n pull-requests: write\n issues: write\n uses: Cysharp/Actions/.github/workflows/stale-issue.yaml@main\n"
},
{
"path": ".gitignore",
"content": "# Build Folders (you can keep bin if you'd like, to store dlls and pdbs)\n[Bb]in/\n[Oo]bj/\n\n# mstest test results\nTestResults\n\n## Ignore Visual Studio temporary files, build results, and\n## files generated by popular Visual Studio add-ons.\n\n# User-specific files\n*.suo\n*.user\n*.sln.docstates\n\n# Build results\n[Dd]ebug/\n[Rr]elease/\nx64/\n*_i.c\n*_p.c\n*.ilk\n*.obj\n*.pch\n*.pdb\n*.pgc\n*.pgd\n*.rsp\n*.sbr\n*.tlb\n*.tli\n*.tlh\n*.tmp\n*.log\n*.vspscc\n*.vssscc\n.builds\n\n# Visual C++ cache files\nipch/\n*.aps\n*.ncb\n*.opensdf\n*.sdf\n\n# Visual Studio profiler\n*.psess\n*.vsp\n*.vspx\n\n# Guidance Automation Toolkit\n*.gpState\n\n# ReSharper is a .NET coding add-in\n_ReSharper*\n\n# NCrunch\n*.ncrunch*\n.*crunch*.local.xml\n\n# Installshield output folder\n[Ee]xpress\n\n# DocProject is a documentation generator add-in\nDocProject/buildhelp/\nDocProject/Help/*.HxT\nDocProject/Help/*.HxC\nDocProject/Help/*.hhc\nDocProject/Help/*.hhk\nDocProject/Help/*.hhp\nDocProject/Help/Html2\nDocProject/Help/html\n\n# Click-Once directory\npublish\n\n# Publish Web Output\n*.Publish.xml\n\n# NuGet Packages Directory\npackages\n\n# Windows Azure Build Output\ncsx\n*.build.csdef\n\n# Windows Store app package directory\nAppPackages/\n\n# Others\n[Bb]in\n[Oo]bj\nsql\nTestResults\n[Tt]est[Rr]esult*\n*.Cache\nClientBin\n[Ss]tyle[Cc]op.*\n~$*\n*.dbmdl\nGenerated_Code #added for RIA/Silverlight projects\n\n# Backup & report files from converting an old project file to a newer\n# Visual Studio version. Backup files are not needed, because we have git ;-)\n_UpgradeReport_Files/\nBackup*/\nUpgradeLog*.XML\n.vs/config/applicationhost.config\n.vs/restore.dg\n\n# OTHER\nnuget/tools/*\n*.nupkg\n\n.vs\n**/.DS_Store\n.idea\n\n# publish directory\nout/\n*.tsbuildinfo\n\n# BenchmarkDotNet Artifacts\nBenchmarkDotNet.Artifacts/\n"
},
{
"path": "Directory.Build.props",
"content": "\n\n \n true\n $(MSBuildThisFileDirectory)opensource.snk\n\n \n false\n $(Version)\n Cysharp\n Cysharp\n © Cysharp, Inc.\n https://github.com/Cysharp/Utf8StreamReader\n README.md\n $(PackageProjectUrl)\n git\n MIT\n Icon.png\n \n\n \n \n \n \n \n\n"
},
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2024 Cysharp, Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "README.md",
"content": "# Utf8StreamReader\n\n[](https://github.com/Cysharp/Utf8StreamReader/actions) [](https://github.com/Cysharp/Utf8StreamReader/releases)\n[](https://nuget.org/packages/Utf8StreamReader)\n\nUtf8 based StreamReader for high performance text processing. In addition to UTF-8 based binary processing, it can also be used as a a high-performance replacement for StreamReader and as a helper for fast binary reading.\n\nAvoiding unnecessary string allocation is a fundamental aspect of recent .NET performance improvements. Given that most file and network data is in UTF8, features like [JsonSerializer](https://learn.microsoft.com/en-us/dotnet/api/system.text.json.jsonserializer?view=net-8.0) and [IUtf8SpanParsable](https://learn.microsoft.com/en-us/dotnet/api/system.iutf8spanparsable-1?view=net-8.0), which operate on UTF8-based data, have been added. More recently, methods like [.NET8 MemoryExtensions.Split](https://learn.microsoft.com/en-us/dotnet/api/system.memoryextensions.split?view=net-8.0), which avoids allocations, have also been introduced.\n\nHowever, for the most common use case of parsing strings delimited by newlines, only the traditional [StreamReader](https://learn.microsoft.com/en-us/dotnet/api/system.io.streamreader) is provided, which generates a new String for each line, resulting in a large amount of allocations.\n\n\n> Read simple 1000000 lines text\n\nIncredibly, there is a **240,000 times** difference!\n\nWhile it is possible to process data in UTF8 format using standard classes like [PipeReader](https://learn.microsoft.com/en-us/dotnet/api/system.io.pipelines.pipereader?view=dotnet-plat-ext-8.0) and [SequenceReader](https://learn.microsoft.com/en-us/dotnet/api/system.buffers.sequencereader-1?view=net-8.0), they are generic librardies, so properly handling newline processing requires considerable effort(Handling BOM and Multiple Types of Newline Characters).\n\n`Utf8StreamReader` provides a familiar API similar to StreamReader, making it easy to use, while its ReadLine-specific implementation maximizes performance.\n\nBy using optimized internal processing, higher performance can be achieved when reading Strings from Files compared to using the standard `StreamReader.ReadToEnd` or `File.ReadAllText` methods.\n\n\n\n> Read from file(1000000 lines text)\n\n```csharp\n[Benchmark]\npublic async Task StreamReaderReadToEndAsync()\n{\n using var sr = new System.IO.StreamReader(filePath);\n return await sr.ReadToEndAsync();\n}\n\n[Benchmark]\npublic async Task Utf8TextReaderReadToEndAsync()\n{\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath).AsTextReader();\n return await sr.ReadToEndAsync();\n}\n\n[Benchmark]\npublic async Task FileReadAllTextAsync()\n{\n return await File.ReadAllTextAsync(filePath);\n}\n```\n\nFor an explanation of the performance difference, please refer to the [ReadString Section](#readstring).\n\n## Getting Started\n\nThis library is distributed via NuGet, supporting `.NET Standard 2.1`, `.NET 6(.NET 7)` and `.NET 8` or above. For information on usage with Unity, please refer to the [Unity Section](#unity).\n\nPM> Install-Package [Utf8StreamReader](https://www.nuget.org/packages/Utf8StreamReader)\n\nThe basic API involves `using var streamReader = new Utf8StreamReader(stream);` and then `ReadOnlyMemory line = await streamReader.ReadLineAsync();`. When enumerating all lines, you can choose from three styles:\n\n```csharp\nusing Cysharp.IO; // namespace of Utf8StreamReader\n\npublic async Task Sample1(Stream stream)\n{\n using var reader = new Utf8StreamReader(stream);\n\n // Most performant style, similar as System.Threading.Channels\n while (await reader.LoadIntoBufferAsync())\n {\n while (reader.TryReadLine(out var line))\n {\n // line is ReadOnlyMemory, deserialize UTF8 directly.\n _ = JsonSerializer.Deserialize(line.Span);\n }\n }\n}\n\npublic async Task Sample2(Stream stream)\n{\n using var reader = new Utf8StreamReader(stream);\n\n // Classical style, same as StreamReader\n ReadOnlyMemory? line = null;\n while ((line = await reader.ReadLineAsync()) != null)\n {\n _ = JsonSerializer.Deserialize(line.Value.Span);\n }\n}\n\npublic async Task Sample3(Stream stream)\n{\n using var reader = new Utf8StreamReader(stream);\n\n // Most easiest style, use async streams\n await foreach (var line in reader.ReadAllLinesAsync())\n {\n _ = JsonSerializer.Deserialize(line.Span);\n }\n}\n```\n\nFrom a performance perspective, `Utf8StreamReader` only provides asynchronous APIs.\n\nTheoretically, the highest performance can be achieved by combining `LoadIntoBufferAsync` and `TryReadLine` in a double while loop. This is similar to the combination of `WaitToReadAsync` and `TryRead` in [Channels](https://learn.microsoft.com/en-us/dotnet/core/extensions/channels).\n\n`ReadLineAsync`, like StreamReader.ReadLine, returns null to indicate that the end has been reached.\n\n`ReadAllLinesAsync` returns an `IAsyncEnumerable>`. Although there is a performance difference, it is minimal, so this API is ideal when you want to use it easily.\n\nAll asynchronous methods accept a `CancellationToken` and support cancellation.\n\nFor a real-world usage example, refer to [StreamMessageReader.cs](https://github.com/Cysharp/Claudia/blob/main/src/Claudia/StreamMessageReader.cs) in [Cysharp/Claudia](https://github.com/Cysharp/Claudia/), a C# SDK for Anthropic Claude, which parses [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events).\n\n## Buffer Lifetimes\n\nThe `ReadOnlyMemory` returned from `ReadLineAsync` or `TryReadLine` is only valid until the next call to `LoadIntoBufferAsync` or `TryReadLine` or `ReadLineAsync`. Since the data is shared with the internal buffer, it may be overwritten, moved, or returned on the next call, so the safety of the data cannot be guaranteed. The received data must be promptly parsed and converted into a separate object. If you want to keep the data as is, use `ToArray()` to convert it to a `byte[]`.\n\nThis design is similar to [System.IO.Pipelines](https://learn.microsoft.com/en-us/dotnet/standard/io/pipelines).\n\n## Read as `ReadOnlyMemory`\n\nYou can convert it to a `Utf8TextReader` that extracts `ReadOnlyMemory` or `string`. Although there is a conversion cost, it is still fast and low allocation, so it can be used as an alternative to `StreamReader`.\n\n\n\nAfter converting with `AsTextReader()`, all the same methods (`TryReadLine`, `ReadLineAsync`, `LoadIntoBufferAsync`, `ReadAllLinesAsync`) can be used.\n\n```csharp\nusing var sr = new Cysharp.IO.Utf8StreamReader(ms).AsTextReader();\nwhile (await sr.LoadIntoBufferAsync())\n{\n while (sr.TryReadLine(out var line))\n {\n // line is ReadOnlyMemory, you can add to StringBuilder or other parsing method.\n\n // If you neeed string, ReadOnlyMemory.ToString() build string instance\n // string str = line.ToString();\n }\n}\n```\n\nYou can perform text processing without allocation, such as splitting `ReadOnlySpan` using [MemoryExtensions.Split](https://learn.microsoft.com/en-us/dotnet/api/system.memoryextensions.split?view=net-8.0#system-memoryextensions-split(system-readonlyspan((system-char))-system-span((system-range))-system-char-system-stringsplitoptions)), and concatenate the results using StringBuilder's [`Append/AppendLine(ReadOnlySpan)`](https://learn.microsoft.com/en-us/dotnet/api/system.text.stringbuilder.append). This way, string-based processing can be done with much lower allocation compared to `StreamReader`.\n\nWhen a string is needed, you can convert `ReadOnlyMemory` to a string using `ToString()`. Even with the added string conversion, the performance is higher than `StreamReader`, so it can be used as a better alternative.\n\n## Optimizing FileStream\n\nSimilar to `StreamReader`, `Utf8StreamReader` has the ability to open a `FileStream` by accepting a `string path`.\n\n```csharp\npublic Utf8StreamReader(string path, FileOpenMode fileOpenMode = FileOpenMode.Throughput)\npublic Utf8StreamReader(string path, int bufferSize, FileOpenMode fileOpenMode = FileOpenMode.Throughput)\npublic Utf8StreamReader(string path, FileStreamOptions options)\npublic Utf8StreamReader(string path, FileStreamOptions options, int bufferSize)\n```\n\nUnfortunately, the `FileStream` used by `StreamReader` is not optimized for modern .NET. For example, when using `FileStream` with asynchronous methods, it should be opened with `useAsync: true` for optimal performance. However, since `StreamReader` has both synchronous and asynchronous methods in its API, false is specified. Additionally, although `StreamReader` itself has a buffer and `FileStream` does not require a buffer, the buffer of `FileStream` is still being utilized.\n\nIt is difficult to handle `FileStream` correctly with high performance. By specifying a `string path`, the stream is opened with options optimized for `Utf8StreamReader`, so it is recommended to use this overload rather than opening `FileStream` yourself. The following is a benchmark of `FileStream`.\n\n\n\n`Utf8StreamReader` opens `FileStream` with the following settings:\n\n```csharp\nvar useAsync = (fileOpenMode == FileOpenMode.Scalability);\nnew FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, useAsync: useAsync)\n```\n\nDue to historical reasons, the options for `FileStream` are odd, but by setting `bufferSize` to 1, you can avoid the use of internal buffers. `FileStream` has been significantly revamped in .NET 6, and by controlling the setting of this option and the way `Utf8StreamReader` is called as a whole, it can function as a thin wrapper around the fast [RandomAccess.ReadAsync](https://learn.microsoft.com/en-us/dotnet/api/system.io.randomaccess.readasync), allowing you to avoid most of the overhead of FileStream.\n\n`FileOpenMode` is a proprietary option of `Utf8StreamReader`.\n\n\n```csharp\npublic enum FileOpenMode\n{\n Scalability,\n Throughput\n}\n```\n\nIn a Windows environment, the table in the [IO section of the Performance Improvements in .NET 6 blog](https://devblogs.microsoft.com/dotnet/performance-improvements-in-net-6/#io) shows that throughput decreases when `useAsync: true` is used.\n\n| Method | Runtime | IsAsync | BufferSize | Mean |\n| - | - | - | - | - |\n| ReadAsync\t| .NET 6.0 | True | 1 | 119.573 ms |\n| ReadAsync\t| .NET 6.0 | False | 1 | 36.018 ms |\n\nBy setting `Utf8StreamReader` to `FileOpenMode.Scalability`, true async I/O is enabled and scalability is prioritized. If set to `FileOpenMode.Throughput`, it internally becomes sync-over-async and consumes the ThreadPool, but reduces the overhead of asynchronous I/O and improves throughput.\n\nIf frequently executed within a server application, setting it to `Scalability`, and for batch applications, setting it to `Throughput` will likely yield the best performance characteristics. The default is `Throughput`. (In the current .NET implementation, both seem to be the same (similar to Throughput on Windows) in Linux environments.)\n\nIn `Utf8StreamReader`, by carefully adjusting the buffer size on the `Utf8StreamReader` side, the performance difference is minimized. Please refer to the above benchmark results image for specific values.\n\nFor overloads that accept `FileStreamOptions`, the above settings are not reflected, so please adjust them manually.\n\n## ReadString\n\nBy combining the above FileStream optimization with `.AsTextReader().ReadToEndAsync()`, you can achieve higher performance when reading out a `string` compared to `StreamReader.ReadToEnd` or `File.ReadAllText`.\n\n\n\nThe implementation of `File.ReadAllText` in dotnet/runtime uses `StreamReader.ReadToEnd`, so they are almost the same. However, in the case of `File.ReadAllText`, it uses `useAsync: true` when opening the `FileStream`. That accounts for the performance difference in the benchmark.\n\nAnother significant difference in the implementation is that `Utf8StreamReader` generates a `string` without using `StringBuilder`. `StreamReader.ReadToEnd` generates a string using the following flow: `byte[] buffer` -> `char[] decodeBuffer` -> `StringBuilder.Append(char[])` -> `StringBuilder.ToString()`, but there are removable inefficiencies. Both `char[]` and `StringBuilder` are `char[]` buffers, and copying occurs. By generating a `string` directly from `char[]`, the copy to the internal buffer of `StringBuilder` can be eliminated.\n\nIn `Utf8StreamReader`'s `.AsTextReader().ReadToEndAsync()`, it receives streaming data in read buffer units from `Utf8StreamReader` (`ReadToEndChunksAsync`), converts it to `char[]` chunks using `Decoder`, and generates the string all at once using `string.Create`.\n\n```csharp\n// Utf8TextReader is a helper class for ReadOnlyMemory and string generation that internally holds Utf8StreamReader\npublic async ValueTask ReadToEndAsync(CancellationToken cancellationToken = default)\n{\n // Using a method similar to .NET 9 LINQ to Objects's ToArray improvement, returns a structure optimized for gap-free sequential expansion\n // StreamReader.ReadToEnd copies the buffer to a StringBuilder, but this implementation holds char[] chunks(char[][]) without copying.\n using var writer = new SegmentedArrayBufferWriter();\n var decoder = Encoding.UTF8.GetDecoder();\n\n // Utf8StreamReader.ReadToEndChunksAsync returns the internal buffer ReadOnlyMemory as an asynchronous sequence upon each read completion\n await foreach (var chunk in reader.ReadToEndChunksAsync(cancellationToken).ConfigureAwait(reader.ConfigureAwait))\n {\n var input = chunk;\n while (input.Length != 0)\n {\n // The Decoder directly writes from the read buffer to the char[] buffer\n decoder.Convert(input.Span, writer.GetMemory().Span, flush: false, out var bytesUsed, out var charsUsed, out var completed);\n input = input.Slice(bytesUsed);\n writer.Advance(charsUsed);\n }\n }\n \n decoder.Convert([], writer.GetMemory().Span, flush: true, out _, out var finalCharsUsed, out _);\n writer.Advance(finalCharsUsed);\n\n // Directly generate a string from the char[][] buffer using String.Create\n return string.Create(writer.WrittenCount, writer, static (stringSpan, writer) =>\n {\n foreach (var item in writer.GetSegmentsAndDispose())\n {\n item.Span.CopyTo(stringSpan);\n stringSpan = stringSpan.Slice(item.Length);\n }\n });\n}\n```\n\nSegmentedArrayBufferWriter borrows the idea (which I proposed) from [the performance improvement of ToArray in LINQ in .NET 9](https://github.com/dotnet/runtime/pull/96570), and internally holds an InlineArray that expands by equal multipliers.\n\n```csharp\n[StructLayout(LayoutKind.Sequential)]\nstruct InlineArray19\n{\n public const int InitialSize = 8192;\n\n T[] array00; // 8192\n T[] array01; // 16384\n T[] array02; // 32768\n T[] array03; // 65536\n T[] array04; // 131072\n T[] array05; // 262144\n T[] array06; // 524288\n T[] array07; // 1048576\n T[] array08; // 2097152\n T[] array09; // 4194304\n T[] array10; // 8388608\n T[] array11; // 16777216\n T[] array12; // 33554432\n T[] array13; // 67108864\n T[] array14; // 134217728\n T[] array15; // 268435456\n T[] array16; // 536870912\n T[] array17; // 1073741824\n T[] array18; // Array.MaxLength - total\n\n public T[] this[int i]\n {\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n get\n {\n if (i < 0 || i > 18) Throw();\n return Unsafe.Add(ref array00, i);\n }\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n set\n {\n if (i < 0 || i > 18) Throw();\n Unsafe.Add(ref array00, i) = value;\n }\n }\n void Throw() { throw new ArgumentOutOfRangeException(); }\n}\n```\n\nWith these optimizations for both reading and writing, we achieved several times the speedup compared to the .NET standard library.\n\n## Binary Read\n\n`TryPeek`, `PeekAsync`, `TryRead`, `ReadAsync`, `TryReadBlock`, and `ReadBlockAsync` enable reading as binary, irrespective of newline codes. For example, [Redis's protocol, RESP](https://redis.io/docs/latest/develop/reference/protocol-spec/), is a text protocol and typically newline-delimited, but after `$N`, it requires reading N bytes (BulkString). For instance, `$5\\r\\nhello\\r\\n` means reading 5 bytes.\n\nHere's an example of how it can be parsed:\n\n```csharp\n// $5\\r\\nhello\\r\\n\nvar line = await reader.ReadLineAsync(); // $5(+ consumed \\r\\n)\nif (line.Value.Span[0] == (byte)'$')\n{\n Utf8Parser.TryParse(line.Value.Span.Slice(1), out int size, out _); // 5\n var block = await reader.ReadBlockAsync(size); // hello\n await reader.ReadLineAsync(); // consume \\r\\n\n Console.WriteLine(Encoding.UTF8.GetString(block.Span));\n}\n```\n\nA sample that parses all RESP code is available in [RespReader.cs](https://github.com/Cysharp/Utf8StreamReader/blob/e400444/sandbox/ConsoleApp1/RespReader.cs).\n\nAdditionally, when using `LoadIntoBufferAsync` and `LoadIntoBufferAtLeastAsync` to include data in the buffer, using `Try***` allows for more efficient execution.\n\n```csharp\nwhile (await reader.LoadIntoBufferAsync())\n{\n while (reader.TryReadLine(out var line))\n {\n switch (line.Span[0])\n {\n case (byte)'$':\n Utf8Parser.TryParse(line.Span.Slice(1), out int size, out _);\n if (!reader.TryReadBlock(size + 2, out var block)) // +2 is \\r\\n\n {\n // ReadBlockAsync is TryReadBlock + LoadIntoBufferAtLeastAsync\n block = await reader.ReadBlockAsync(size + 2);\n }\n yield return block.Slice(0, size);\n break;\n // and others('+', '-', ':', '*')\n default:\n break;\n }\n }\n}\n```\n\nWhen using `ReadToEndAsync`, you can obtain a `byte[]` using Utf8StreamReader's efficient binary reading/concatenation (`SegmentedArrayBufferWriter, InlineArray19`). \n\n```csharp\nusing var reader = new Utf8StreamReader(stream);\nbyte[] bytes = await reader.ReadToEndAsync();\n```\n\n`ReadToEndAsync()` has two optional overloads, `(bool disableBomCheck)` and `(long resultSizeHint)`. \n\nIf `disableBomCheck` is true, it disables the BOM check/trim and always performs a complete binary-matching read. The default for `ReadToEndAsync` is true, which always expects a binary-matching read. If false, it follows Utf8StreamReader.SkipBom.\n\n`resultSizeHint` allows for reducing the copy cost by directly generating `new byte[resultSizeHint]` when the final binary size is known and reading directly into that buffer. When reading a file, i.e., when the `Stream` is a `FileStream` and seekable, `FileStream.Length` is used as the resultSizeHint as an optimization.\n\nHere is the peformance comparison between copying a normal `Stream` to a `MemoryStream` by `CopyToAsync` and using `ToArray`, and using `ReadToEndAsync` of `Utf8StreamReader` when converting to `byte[]`. The options are adjusted so that optimization does not occur when directly passing FileStream to Utf8StreamReader, in order to intentionally avoid optimization.\n\n\n\n```csharp\n[Benchmark]\npublic async Task MemoryStreamCopyToToArray()\n{\n using var fs = new FileStream(filePath, FileMode.Open);\n var ms = new MemoryStream();\n await fs.CopyToAsync(ms);\n\n return ms.ToArray();\n}\n\n[Benchmark]\npublic async Task Utf8StreamReaderReadToEndAsync()\n{\n using var fs = new FileStream(filePath, FileMode.Open);\n using var sr = new Cysharp.IO.Utf8StreamReader(fs);\n return await sr.ReadToEndAsync(disableBomCheck: false); // hack for disable optimize(for benchmark fairness)\n}\n```\n\n## Reset\n\n`Utf8StreamReader` is a class that supports reuse. By calling `Reset()`, the Stream and internal state are released. Using `Reset(Stream)`, it can be reused with a new `Stream`.\n\n## Options\n\nThe constructor accepts `int bufferSize` and `bool leaveOpen` as parameters.\n\n`int bufferSize` defaults to 65536 and the buffer is rented from `ArrayPool`. If the data per line is large, changing the buffer size may improve performance. When the buffer size and the size per line are close, frequent buffer copy operations occur, leading to performance degradation.\n\n`bool leaveOpen` determines whether the internal Stream is also disposed when the object is disposed. The default is `false`, which means the Stream is disposed.\n\nAdditionally, there are init properties that allow changing the option values for `ConfigureAwait`, `SyncRead` and `SkipBom`.\n\n`bool ConfigureAwait { init; }` allows you to specify the value for `ConfigureAwait(bool continueOnCapturedContext)` when awaiting asynchronous methods internally. The default is `false`.\n\n`bool SyncRead { init; }` configures the Stream to use synchronous reading, meaning it will use Read instead. This causes all Async operations to complete synchronously. There is potential for slight performance improvements when a `FileStream` is opened with `useAsync:false`. Normally, leaving it as false is fine. The default is `false`.\n\n`bool SkipBom { init; }` determines whether to identify and skip the BOM (Byte Order Mark) included at the beginning of the data during the first read. The default is `true`, which means the BOM is skipped.\n\nCurrently, this is not an option, but `Utf8StreamReader` only determines `CRLF(\\r\\n)` or `LF(\\n)` as newline characters. Since environments that use `CR(\\r)` are now extremely rare, the CR check is omitted for performance reasons. If you need this functionality, please let us know by creating an Issue. We will consider adding it as an option\n\nUnity\n---\nUnity, which supports .NET Standard 2.1, can run this library. Since the library is only provided through NuGet, it is recommended to use [NuGetForUnity](https://github.com/GlitchEnzo/NuGetForUnity) for installation.\n\nFor detailed instructions on using NuGet libraries in Unity, please refer to the documentation of [Cysharp/R3](https://github.com/Cysharp/R3/) and other similar resources.\n\nLicense\n---\nThis library is under the MIT License.\n"
},
{
"path": "Utf8StreamReader.sln",
"content": "\nMicrosoft Visual Studio Solution File, Format Version 12.00\n# Visual Studio Version 17\nVisualStudioVersion = 17.8.34330.188\nMinimumVisualStudioVersion = 10.0.40219.1\nProject(\"{2150E333-8FDC-42A3-9474-1A3956D46DE8}\") = \"src\", \"src\", \"{BD07BD08-1CB4-41AE-B2BD-3975BE13B8EC}\"\nEndProject\nProject(\"{9A19103F-16F7-4668-BE54-9A1E7A4F7556}\") = \"Utf8StreamReader\", \"src\\Utf8StreamReader\\Utf8StreamReader.csproj\", \"{983561F1-F180-4188-AE80-BFA95FD69656}\"\nEndProject\nProject(\"{2150E333-8FDC-42A3-9474-1A3956D46DE8}\") = \"tests\", \"tests\", \"{5A8808D6-63E0-48EE-A115-0380E0E57156}\"\nEndProject\nProject(\"{9A19103F-16F7-4668-BE54-9A1E7A4F7556}\") = \"Utf8StreamReader.Tests\", \"tests\\Utf8StreamReader.Tests\\Utf8StreamReader.Tests.csproj\", \"{6C953584-A04B-42C7-9CF3-267AFB010C2B}\"\nEndProject\nProject(\"{2150E333-8FDC-42A3-9474-1A3956D46DE8}\") = \"sandbox\", \"sandbox\", \"{6BA94544-B2DF-4DD2-9390-DAA8AF5CA90A}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"ConsoleApp1\", \"sandbox\\ConsoleApp1\\ConsoleApp1.csproj\", \"{27B89B32-EC1A-48B0-BFC9-6172FCCE2961}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"Benchmark\", \"sandbox\\Benchmark\\Benchmark.csproj\", \"{48293CC8-A87C-4F59-A398-51CD37E6B62B}\"\nEndProject\nGlobal\n\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\n\t\tDebug|Any CPU = Debug|Any CPU\n\t\tRelease|Any CPU = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\n\t\t{983561F1-F180-4188-AE80-BFA95FD69656}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{983561F1-F180-4188-AE80-BFA95FD69656}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{983561F1-F180-4188-AE80-BFA95FD69656}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{983561F1-F180-4188-AE80-BFA95FD69656}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{6C953584-A04B-42C7-9CF3-267AFB010C2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{6C953584-A04B-42C7-9CF3-267AFB010C2B}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{6C953584-A04B-42C7-9CF3-267AFB010C2B}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{6C953584-A04B-42C7-9CF3-267AFB010C2B}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{27B89B32-EC1A-48B0-BFC9-6172FCCE2961}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{27B89B32-EC1A-48B0-BFC9-6172FCCE2961}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{27B89B32-EC1A-48B0-BFC9-6172FCCE2961}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{27B89B32-EC1A-48B0-BFC9-6172FCCE2961}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{48293CC8-A87C-4F59-A398-51CD37E6B62B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{48293CC8-A87C-4F59-A398-51CD37E6B62B}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{48293CC8-A87C-4F59-A398-51CD37E6B62B}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{48293CC8-A87C-4F59-A398-51CD37E6B62B}.Release|Any CPU.Build.0 = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(SolutionProperties) = preSolution\n\t\tHideSolutionNode = FALSE\n\tEndGlobalSection\n\tGlobalSection(NestedProjects) = preSolution\n\t\t{983561F1-F180-4188-AE80-BFA95FD69656} = {BD07BD08-1CB4-41AE-B2BD-3975BE13B8EC}\n\t\t{6C953584-A04B-42C7-9CF3-267AFB010C2B} = {5A8808D6-63E0-48EE-A115-0380E0E57156}\n\t\t{27B89B32-EC1A-48B0-BFC9-6172FCCE2961} = {6BA94544-B2DF-4DD2-9390-DAA8AF5CA90A}\n\t\t{48293CC8-A87C-4F59-A398-51CD37E6B62B} = {6BA94544-B2DF-4DD2-9390-DAA8AF5CA90A}\n\tEndGlobalSection\n\tGlobalSection(ExtensibilityGlobals) = postSolution\n\t\tSolutionGuid = {38C0CA37-B15E-4200-9F2C-AD08076E4013}\n\tEndGlobalSection\nEndGlobal\n"
},
{
"path": "sandbox/Benchmark/Benchmark.csproj",
"content": "\n\n \n Exe\n net8.0\n enable\n enable\n \n\n \n \n \n \n\n \n \n \n\n"
},
{
"path": "sandbox/Benchmark/BytesReadToEnd.cs",
"content": "using BenchmarkDotNet.Attributes;\nusing Cysharp.IO;\nusing System.Text.Encodings.Web;\nusing System.Text.Json;\nusing System.Text.Unicode;\n\nnamespace Benchmark;\n\n[SimpleJob, MemoryDiagnoser]\npublic class BytesReadToEnd\n{\n const int C = 1000000;\n\n string filePath = default!;\n\n [GlobalSetup]\n public void GlobalSetup()\n {\n var options = new JsonSerializerOptions\n {\n Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)\n };\n\n var path = Path.GetTempFileName();\n var newline = OperatingSystem.IsWindows() ? \"\\r\\n\"u8 : \"\\n\"u8;\n using var file = File.OpenWrite(path);\n for (var i = 0; i < C; i++)\n {\n var json = JsonSerializer.SerializeToUtf8Bytes(\n new MyClass { MyProperty = i, MyProperty2 = \"あいうえおかきくけこ\" }, options);\n file.Write(json);\n file.Write(newline);\n }\n\n filePath = path;\n }\n\n [GlobalCleanup]\n public void GlobalCleanup()\n {\n File.Delete(filePath);\n }\n\n [Benchmark]\n public async Task FileReadAllBytesAsync()\n {\n // ReadAllBytes knows file-length so fastest.\n return await File.ReadAllBytesAsync(filePath);\n }\n\n [Benchmark]\n public async Task Utf8StreamReaderReadToEndAsync()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath);\n return await sr.ReadToEndAsync();\n }\n}\n\n[SimpleJob, MemoryDiagnoser]\npublic class BytesReadToEnd2\n{\n const int C = 1000000;\n\n string filePath = default!;\n\n [GlobalSetup]\n public void GlobalSetup()\n {\n var options = new JsonSerializerOptions\n {\n Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)\n };\n\n var path = Path.GetTempFileName();\n var newline = OperatingSystem.IsWindows() ? \"\\r\\n\"u8 : \"\\n\"u8;\n using var file = File.OpenWrite(path);\n for (var i = 0; i < C; i++)\n {\n var json = JsonSerializer.SerializeToUtf8Bytes(\n new MyClass { MyProperty = i, MyProperty2 = \"あいうえおかきくけこ\" }, options);\n file.Write(json);\n file.Write(newline);\n }\n\n filePath = path;\n }\n\n [GlobalCleanup]\n public void GlobalCleanup()\n {\n File.Delete(filePath);\n }\n\n [Benchmark]\n public async Task MemoryStreamCopyToToArray()\n {\n using var fs = new FileStream(filePath, FileMode.Open);\n var ms = new MemoryStream();\n await fs.CopyToAsync(ms);\n\n return ms.ToArray();\n }\n\n [Benchmark]\n public async Task Utf8StreamReaderReadToEndAsync()\n {\n using var fs = new FileStream(filePath, FileMode.Open);\n using var sr = new Cysharp.IO.Utf8StreamReader(fs);\n return await sr.ReadToEndAsync(disableBomCheck: false); // hack for ignore optimize(for benchmark fairness)\n }\n}\n"
},
{
"path": "sandbox/Benchmark/FromFile.cs",
"content": "using BenchmarkDotNet.Attributes;\nusing Cysharp.IO;\nusing System.Text;\nusing System.Text.Encodings.Web;\nusing System.Text.Json;\nusing System.Text.Unicode;\n\nnamespace Benchmark;\n\n[SimpleJob, MemoryDiagnoser]\npublic class FromFile\n{\n const int C = 1000000;\n\n string filePath = default!;\n\n [GlobalSetup]\n public void GlobalSetup()\n {\n var options = new JsonSerializerOptions\n {\n Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)\n };\n\n var path = Path.GetTempFileName();\n var newline = OperatingSystem.IsWindows() ? \"\\r\\n\"u8 : \"\\n\"u8;\n using var file = File.OpenWrite(path);\n for (var i = 0; i < C; i++)\n {\n var json = JsonSerializer.SerializeToUtf8Bytes(\n new MyClass { MyProperty = i, MyProperty2 = \"あいうえおかきくけこ\" }, options);\n file.Write(json);\n file.Write(newline);\n }\n\n filePath = path;\n }\n\n [GlobalCleanup]\n public void GlobalCleanup()\n {\n File.Delete(filePath);\n }\n\n [Benchmark]\n public async Task StreamReaderFileStream()\n {\n using var sr = new System.IO.StreamReader(filePath);\n string? line;\n while ((line = await sr.ReadLineAsync()) != null)\n {\n // ...\n }\n }\n\n [Benchmark]\n public async Task FileReadLinesAsync()\n {\n await foreach (var line in File.ReadLinesAsync(filePath, Encoding.UTF8))\n {\n }\n }\n\n [Benchmark]\n public async Task Utf8StreamReaderFileStreamScalability()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Scalability);\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // ...\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8StreamReaderFileStreamThroughput()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Throughput);\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // ...\n }\n }\n }\n\n [Benchmark]\n public async ValueTask Utf8StreamReaderFileStreamThroughputSyncRead()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Throughput) { SyncRead = true };\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReaderFileStreamScalability()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Scalability).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // ...\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReaderFileStreamThroughput()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Throughput).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // ...\n }\n }\n }\n\n [Benchmark]\n public async ValueTask Utf8TextReaderFileStreamThroughputSyncRead()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Throughput) { SyncRead = true }.AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // ...\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReaderToStringFileStreamScalability()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Scalability).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n _ = line.ToString();\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReaderToStringFileStreamThroughput()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath, fileOpenMode: FileOpenMode.Throughput).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n _ = line.ToString();\n }\n }\n }\n}\n"
},
{
"path": "sandbox/Benchmark/FromMemory.cs",
"content": "using System.Buffers;\nusing System.IO.Pipelines;\nusing System.Text;\nusing System.Text.Encodings.Web;\nusing System.Text.Json;\nusing System.Text.Unicode;\nusing BenchmarkDotNet.Attributes;\nusing Cysharp.IO;\n\nnamespace Benchmark;\n\n[SimpleJob, MemoryDiagnoser]\npublic class FromMemory\n{\n const int C = 1000000;\n // const int C = 100;\n\n byte[] utf8Data = default!;\n MemoryStream ms = default!;\n\n [GlobalSetup]\n public void GlobalSetup()\n {\n var options = new JsonSerializerOptions\n {\n Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)\n };\n\n var jsonLines = Enumerable.Range(0, C)\n .Select(x => new MyClass { MyProperty = x, MyProperty2 = \"あいうえおかきくけこ\" })\n .Select(x => JsonSerializer.Serialize(x, options))\n .ToArray();\n\n utf8Data = Encoding.UTF8.GetBytes(string.Join(Environment.NewLine, jsonLines));\n }\n\n [IterationSetup]\n public void Setup()\n {\n ms = new MemoryStream(utf8Data);\n }\n\n [Benchmark]\n public async Task StreamReader()\n {\n using var sr = new System.IO.StreamReader(ms);\n string? line;\n while ((line = await sr.ReadLineAsync()) != null)\n {\n // Console.WriteLine(line);\n }\n }\n\n [Benchmark]\n public async Task Utf8StreamReader()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(ms);\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // Console.WriteLine(Encoding.UTF8.GetString( line.Span));\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReader()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(ms).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n // Console.WriteLine(Encoding.UTF8.GetString( line.Span));\n }\n }\n }\n\n [Benchmark]\n public async Task Utf8TextReaderToString()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(ms).AsTextReader();\n while (await sr.LoadIntoBufferAsync())\n {\n while (sr.TryReadLine(out var line))\n {\n _ = line.ToString();\n // Console.WriteLine(Encoding.UTF8.GetString( line.Span));\n }\n }\n }\n\n //[Benchmark]\n //public async Task Utf8StreamReaderReadLine()\n //{\n // using var sr = new Cysharp.IO.Utf8StreamReader(ms);\n // ReadOnlyMemory? line;\n // while ((line = await sr.ReadLineAsync()) != null)\n // {\n // // Console.WriteLine(Encoding.UTF8.GetString(line.Value.Span));\n // }\n //}\n\n //[Benchmark]\n //public async Task Utf8StreamReaderReadAllLines()\n //{\n // using var sr = new Cysharp.IO.Utf8StreamReader(ms);\n // await foreach (var line in sr.ReadAllLinesAsync())\n // {\n // //Console.WriteLine(Encoding.UTF8.GetString(line.Span));\n // }\n //}\n\n [Benchmark]\n public async Task PipeReaderSequenceReader()\n {\n using (ms)\n {\n var reader = PipeReader.Create(ms);\n\n READ_AGAIN:\n var readResult = await reader.ReadAsync();\n\n if (!(readResult.IsCompleted | readResult.IsCanceled))\n {\n var buffer = readResult.Buffer;\n\n while (TryReadData(ref buffer, out var line))\n {\n //Console.WriteLine(Encoding.UTF8.GetString(line));\n }\n\n reader.AdvanceTo(buffer.Start, buffer.End);\n goto READ_AGAIN;\n }\n\n }\n\n static bool TryReadData(ref ReadOnlySequence buffer, out ReadOnlySequence line)\n {\n var reader = new SequenceReader(buffer);\n if (reader.TryReadTo(out line, (byte)'\\n', advancePastDelimiter: true))\n {\n buffer = buffer.Slice(reader.Consumed);\n return true;\n }\n return false;\n }\n }\n\n //[Benchmark]\n //public async Task PipelineStreamReader2()\n //{\n // using (ms)\n // {\n // var reader = PipeReader.Create(ms);\n\n // READ_AGAIN:\n // var readResult = await reader.ReadAsync();\n\n // if (!(readResult.IsCompleted | readResult.IsCanceled))\n // {\n // var buffer = readResult.Buffer;\n // ConsumeAllData(ref buffer);\n // reader.AdvanceTo(buffer.Start, buffer.End);\n // goto READ_AGAIN;\n // }\n // }\n\n // static void ConsumeAllData(ref ReadOnlySequence buffer)\n // {\n // var reader = new SequenceReader(buffer);\n // while (reader.TryReadTo(out ReadOnlySequence line, (byte)'\\n', advancePastDelimiter: true))\n // {\n // //Console.WriteLine(Encoding.UTF8.GetString(line));\n // }\n // buffer = buffer.Slice(reader.Consumed);\n // }\n //}\n}\n\n\npublic class MyClass\n{\n public int MyProperty { get; set; }\n public string? MyProperty2 { get; set; }\n}\n"
},
{
"path": "sandbox/Benchmark/Program.cs",
"content": "#if DEBUG\n\nusing Benchmark;\nusing System.Runtime.CompilerServices;\n\nglobal::System.Console.WriteLine(\"DEBUG\");\n\n//var benchmark = new BytesReadToEnd();\nvar benchmark = new ReadToEndString();\nbenchmark.GlobalSetup();\n\n//var s1 = await benchmark.FileReadAllBytesAsync();\nvar s2 = await benchmark.Utf8TextReaderReadToEndAsync();\n\n//Console.WriteLine(s1.SequenceEqual(s2));\n\nbenchmark.GlobalCleanup();\n\n#else\nusing BenchmarkDotNet.Running;\n\nBenchmarkSwitcher\n .FromAssembly(typeof(Program).Assembly)\n .Run(args);\n\n#endif\n"
},
{
"path": "sandbox/Benchmark/ReadToEndString.cs",
"content": "using BenchmarkDotNet.Attributes;\nusing Cysharp.IO;\nusing System.Text.Encodings.Web;\nusing System.Text.Json;\nusing System.Text.Unicode;\n\nnamespace Benchmark;\n\n[SimpleJob, MemoryDiagnoser]\npublic class ReadToEndString\n{\n const int C = 1000000;\n\n string filePath = default!;\n\n [GlobalSetup]\n public void GlobalSetup()\n {\n var options = new JsonSerializerOptions\n {\n Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)\n };\n\n var path = Path.GetTempFileName();\n var newline = OperatingSystem.IsWindows() ? \"\\r\\n\"u8 : \"\\n\"u8;\n using var file = File.OpenWrite(path);\n for (var i = 0; i < C; i++)\n {\n var json = JsonSerializer.SerializeToUtf8Bytes(\n new MyClass { MyProperty = i, MyProperty2 = \"あいうえおかきくけこ\" }, options);\n file.Write(json);\n file.Write(newline);\n }\n\n filePath = path;\n }\n\n [GlobalCleanup]\n public void GlobalCleanup()\n {\n File.Delete(filePath);\n }\n\n [Benchmark]\n public async Task StreamReaderReadToEndAsync()\n {\n using var sr = new System.IO.StreamReader(filePath);\n return await sr.ReadToEndAsync();\n }\n\n [Benchmark]\n public async Task Utf8TextReaderReadToEndAsync()\n {\n using var sr = new Cysharp.IO.Utf8StreamReader(filePath).AsTextReader();\n return await sr.ReadToEndAsync();\n }\n\n [Benchmark]\n public async Task FileReadAllTextAsync()\n {\n return await File.ReadAllTextAsync(filePath);\n }\n\n}\n"
},
{
"path": "sandbox/ConsoleApp1/ConsoleApp1.csproj",
"content": "\n\n \n Exe\n net8.0\n enable\n enable\n \n\n \n \n \n\n \n \n \n\n \n \n Always\n \n \n\n\n"
},
{
"path": "sandbox/ConsoleApp1/Program.cs",
"content": "using Cysharp.IO;\nusing Microsoft.Win32.SafeHandles;\nusing System.Buffers;\nusing System.Buffers.Text;\nusing System.IO;\nusing System.IO.Pipelines;\nusing System.Runtime.InteropServices;\nusing System.Runtime.InteropServices.Marshalling;\nusing System.Text;\nusing System.Text.Encodings.Web;\nusing System.Text.Json;\nusing System.Text.Unicode;\n\n\n\n\nvar aa = Encoding.UTF8.GetBytes(\"$5\\r\\nhello\\r\\n\");\nvar stream = new MemoryStream(aa);\n\nusing var reader = new Utf8StreamReader(stream) { SkipBom = false };\nbyte[] bytes = await reader.ReadToEndAsync();\n\n\n//while (await reader.LoadIntoBufferAsync())\n//{\n// while (reader.TryReadLine(out var line))\n// {\n// switch (line.Span[0])\n// {\n// case (byte)'$':\n// Utf8Parser.TryParse(line.Span.Slice(1), out int size, out _);\n// if (!reader.TryReadBlock(size + 2, out var block)) // +2 is \\r\\n\n// {\n// // ReadBlockAsync is TryReadBlock + LoadIntoBufferAtLeastAsync\n// block = await reader.ReadBlockAsync(size + 2);\n// }\n// yield return block.Slice(0, size);\n// break;\n// // and others('+', '-', ':', '*')\n// default:\n// break;\n// }\n// }\n//}\n\n\n//var path = \"file1.txt\";\n\n\n//var fs = new FileStream(path, FileMode.Open,FileAccess.Read, FileShare.Read, 0, false);\n//var buf = new byte[1024];\n//await fs.ReadAsync(buf);\n\n//using var reader = new Utf8StreamReader(path).AsTextReader();\n\n\n\n//var str = await reader.ReadToEndAsync();\n//Console.WriteLine(str.ToString());\n\n// new StreamReader().ReadBlock(\n\n\n//var options = new JsonSerializerOptions();\n//options.Encoder = JavaScriptEncoder.Create(UnicodeRanges.All);\n\n//var jsonLines = Enumerable.Range(0, 100000)\n// .Select(x => new MyClass { MyProperty = x, MyProperty2 = \"あいうえおかきくけこ\" })\n// .Select(x => JsonSerializer.Serialize(x, options))\n// .ToArray();\n\n//var utf8Data = Encoding.UTF8.GetBytes(string.Join(Environment.NewLine, jsonLines));\n\n//var ms = new MemoryStream(utf8Data);\n\n\n////using var sr = new System.IO.StreamReader(ms);\n////string? line;\n////while ((line = await sr.ReadLineAsync()) != null)\n////{\n//// // JsonSerializer.Deserialize(line);\n////}\n\n//using var sr = new Cysharp.IO.Utf8StreamReader(ms);\n//ReadOnlyMemory? line;\n//while ((line = await sr.ReadLineAsync()) != null)\n//{\n//}\n\n\n\n//public class MyClass\n//{\n// public int MyProperty { get; set; }\n// public string? MyProperty2 { get; set; }\n//}\n\n"
},
{
"path": "sandbox/ConsoleApp1/ReadMeSample.cs",
"content": "using Cysharp.IO;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Text.Json;\nusing System.Threading.Tasks;\n\nnamespace ConsoleApp1;\n\ninternal class ReadMeSample\n{\n\n\n public async void Sample1(Stream stream)\n {\n using var reader = new Utf8StreamReader(stream);\n\n // Most performant style, similar as System.Threading.Channels\n while (await reader.LoadIntoBufferAsync())\n {\n while (reader.TryReadLine(out var line))\n {\n // line is ReadOnlyMemory, deserialize UTF8 directly.\n _ = JsonSerializer.Deserialize(line.Span);\n }\n }\n }\n\n public async void Sample2(Stream stream)\n {\n using var reader = new Utf8StreamReader(stream);\n\n // Classical style, same as StreamReader\n ReadOnlyMemory? line = null;\n while ((line = await reader.ReadLineAsync()) != null)\n {\n _ = JsonSerializer.Deserialize(line.Value.Span);\n }\n }\n\n public async void Sample3(Stream stream)\n {\n using var reader = new Utf8StreamReader(stream);\n\n // Most easiest style, use async streams\n await foreach (var line in reader.ReadAllLinesAsync())\n {\n _ = JsonSerializer.Deserialize(line.Span);\n }\n }\n}\n\n\npublic class Foo\n{\n\n}\n"
},
{
"path": "sandbox/ConsoleApp1/RespReader.cs",
"content": "using Cysharp.IO;\nusing System.Buffers.Text;\nusing System.Text;\n\nnamespace ConsoleApp1;\n\npublic enum RespType : byte\n{\n SimpleStrings = (byte)'+',\n Errors = (byte)'-',\n Integers = (byte)':',\n BulkStrings = (byte)'$',\n Arrays = (byte)'*'\n}\n\npublic class RespReader : IDisposable\n{\n Utf8StreamReader reader;\n\n public RespReader(Stream stream)\n {\n this.reader = new Utf8StreamReader(stream);\n }\n\n // NOTE: for more fast processing, you need to use TryRead method.\n\n public async ValueTask ReadRespTypeAsync(CancellationToken cancellationToken = default)\n {\n return (RespType)await reader.ReadAsync(cancellationToken);\n }\n\n // all read message api expect befor call ReadRespTypeAsync(already trimed type prefix)\n\n public async ValueTask ReadSimpleStringAsync(CancellationToken cancellationToken = default) // +OK\\r\\n\n {\n return Encoding.UTF8.GetString((await reader.ReadLineAsync(cancellationToken)).Value.Span);\n }\n\n public async ValueTask ReadErrorMessageAsync(CancellationToken cancellationToken = default) // -Error message\\r\\n\n {\n return Encoding.UTF8.GetString((await reader.ReadLineAsync(cancellationToken)).Value.Span);\n }\n\n public async ValueTask ReadIntegerAsync(CancellationToken cancellationToken = default) // :1000\\r\\n\n {\n var line = await reader.ReadLineAsync(cancellationToken);\n Utf8Parser.TryParse(line.Value.Span, out long value, out _);\n return value;\n }\n\n public async ValueTask?> ReadBulkStringAsync(CancellationToken cancellationToken = default) // \"$5\\r\\nhello\\r\\n\"\n {\n var line = await reader.ReadLineAsync(cancellationToken);\n Utf8Parser.TryParse(line.Value.Span, out int count, out _);\n if (count == -1)\n {\n return null;\n }\n else\n {\n var dataWithNewLine = await reader.ReadBlockAsync(count + 2, cancellationToken);\n return dataWithNewLine[..^2]; // without newline\n }\n }\n\n // for perf improvement, ReadIntegerArray, ReadStringArray, ReadArray for bulkstrings is better approach\n public async ValueTask ReadArrayAsync(CancellationToken cancellationToken = default) // \"*2\\r\\n$5\\r\\nhello\\r\\n$5\\r\\nworld\\r\\n\"\n {\n var line = await reader.ReadLineAsync();\n Utf8Parser.TryParse(line.Value.Span, out int count, out _);\n\n var result = new object[count];\n for (int i = 0; i < count; i++)\n {\n var type = await ReadRespTypeAsync(cancellationToken);\n switch (type)\n {\n case RespType.SimpleStrings:\n result[i] = await ReadSimpleStringAsync(cancellationToken);\n break;\n case RespType.Errors:\n result[i] = await ReadErrorMessageAsync(cancellationToken);\n break;\n case RespType.Integers:\n result[i] = await ReadIntegerAsync(cancellationToken);\n break;\n case RespType.BulkStrings:\n result[i] = (await ReadBulkStringAsync(cancellationToken)).Value.ToArray(); // materialize immediately\n break;\n case RespType.Arrays:\n result[i] = await ReadArrayAsync(cancellationToken);\n break;\n default:\n break;\n }\n }\n\n return result;\n }\n\n public void Dispose()\n {\n reader.Dispose();\n }\n}\n"
},
{
"path": "sandbox/ConsoleApp1/file1.txt",
"content": "abcde\nfgh\nijklmnopqrs\n"
},
{
"path": "src/Utf8StreamReader/SegmentedArrayBufferWriter.cs",
"content": "using System.Buffers;\nusing System.Runtime.CompilerServices;\nusing System.Runtime.InteropServices;\n\nnamespace Cysharp.IO;\n\n// similar as .NET9 SegmentedArrayBuilder but for async operation and direct write\ninternal sealed class SegmentedArrayBufferWriter : IDisposable\n{\n // NetStandard2.1 does not have Array.MaxLength so use constant.\n const int ArrayMaxLength = 0X7FFFFFC7;\n\n InlineArray19 segments;\n int currentSegmentIndex;\n int countInFinishedSegments;\n\n T[] currentSegment;\n int currentWritten;\n\n bool isDisposed = false;\n\n public int WrittenCount => countInFinishedSegments + currentWritten;\n\n public SegmentedArrayBufferWriter()\n {\n currentSegment = segments[0] = ArrayPool.Shared.Rent(InlineArray19.InitialSize);\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n public Memory GetMemory() // no sizeHint\n {\n return currentSegment.AsMemory(currentWritten);\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n public Span GetSpan()\n {\n return currentSegment.AsSpan(currentWritten);\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n public void Advance(int count)\n {\n checked\n {\n currentWritten += count;\n };\n if (currentWritten == currentSegment.Length)\n {\n AllocateNextMemory();\n }\n }\n\n void AllocateNextMemory()\n {\n countInFinishedSegments += currentSegment.Length;\n var nextSize = currentSegment.Length * 2L;\n if (nextSize + countInFinishedSegments > ArrayMaxLength)\n {\n nextSize = ArrayMaxLength - countInFinishedSegments;\n }\n\n currentSegmentIndex++;\n currentSegment = segments[currentSegmentIndex] = ArrayPool.Shared.Rent((int)nextSize);\n currentWritten = 0;\n }\n\n public void Write(ReadOnlySpan source)\n {\n while (source.Length != 0)\n {\n var destination = GetSpan();\n var copySize = Math.Min(source.Length, destination.Length);\n\n source.Slice(0, copySize).CopyTo(destination);\n\n Advance(copySize);\n source = source.Slice(copySize);\n }\n }\n\n public T[] ToArrayAndDispose()\n {\n if (isDisposed) throw new ObjectDisposedException(\"\");\n isDisposed = true;\n\n var size = checked(countInFinishedSegments + currentWritten);\n if (size == 0)\n {\n ArrayPool.Shared.Return(currentSegment, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n return [];\n }\n\n#if !NETSTANDARD\n var result = GC.AllocateUninitializedArray(size);\n#else\n var result = new T[size];\n#endif\n var destination = result.AsSpan();\n\n // without current\n for (int i = 0; i < currentSegmentIndex; i++)\n {\n var segment = segments[i];\n segment.AsSpan().CopyTo(destination);\n destination = destination.Slice(segment.Length);\n ArrayPool.Shared.Return(segment, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n }\n\n // write current\n currentSegment.AsSpan(0, currentWritten).CopyTo(destination);\n ArrayPool.Shared.Return(currentSegment, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n\n currentSegment = null!;\n segments = default!;\n return result;\n }\n\n // NOTE: create struct enumerator?\n public IEnumerable> GetSegmentsAndDispose()\n {\n if (isDisposed) throw new ObjectDisposedException(\"\");\n isDisposed = true;\n\n // without current\n for (int i = 0; i < currentSegmentIndex; i++)\n {\n var segment = segments[i];\n yield return segment;\n ArrayPool.Shared.Return(segment, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n }\n\n // current\n if (currentWritten != 0)\n {\n yield return currentSegment.AsMemory(0, currentWritten);\n }\n ArrayPool.Shared.Return(currentSegment, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n\n currentSegment = null!;\n segments = default!;\n }\n\n public void Dispose()\n {\n if (isDisposed) return;\n\n isDisposed = true;\n for (int i = 0; i <= currentSegmentIndex; i++)\n {\n ArrayPool.Shared.Return(segments[i], clearArray: RuntimeHelpers.IsReferenceOrContainsReferences());\n }\n\n currentSegment = null!;\n segments = default!;\n }\n}\n\n[StructLayout(LayoutKind.Sequential)]\nstruct InlineArray19\n{\n public const int InitialSize = 8192;\n\n T[] array00; // 8192\n T[] array01; // 16384\n T[] array02; // 32768\n T[] array03; // 65536\n T[] array04; // 131072\n T[] array05; // 262144\n T[] array06; // 524288\n T[] array07; // 1048576\n T[] array08; // 2097152\n T[] array09; // 4194304\n T[] array10; // 8388608\n T[] array11; // 16777216\n T[] array12; // 33554432\n T[] array13; // 67108864\n T[] array14; // 134217728\n T[] array15; // 268435456\n T[] array16; // 536870912\n T[] array17; // 1073741824\n T[] array18; // Array.MaxLength - total\n\n public T[] this[int i]\n {\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n get\n {\n if (i < 0 || i > 18) Throw();\n return Unsafe.Add(ref array00, i);\n }\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n set\n {\n if (i < 0 || i > 18) Throw();\n Unsafe.Add(ref array00, i) = value;\n }\n }\n\n void Throw()\n {\n throw new ArgumentOutOfRangeException();\n }\n}\n"
},
{
"path": "src/Utf8StreamReader/Utf8StreamReader.cs",
"content": "using System.Buffers;\nusing System.Runtime.CompilerServices;\nusing System.Text;\n\nnamespace Cysharp.IO;\n\npublic enum FileOpenMode\n{\n Scalability,\n Throughput\n}\n\npublic sealed class Utf8StreamReader : IAsyncDisposable, IDisposable\n{\n // NetStandard2.1 does not have Array.MaxLength so use constant.\n const int ArrayMaxLength = 0X7FFFFFC7;\n\n const int DefaultBufferSize = 65536;\n const int MinBufferSize = 1024;\n\n Stream stream;\n readonly bool leaveOpen;\n readonly int bufferSize;\n bool endOfStream;\n bool checkPreamble = true;\n bool skipBom = true;\n bool isDisposed;\n\n byte[] inputBuffer;\n int positionBegin;\n int positionEnd;\n int lastNewLinePosition = -2; // -2 is not exists new line in buffer, -1 is not yet searched. absolute path from inputBuffer begin\n int lastExaminedPosition;\n\n public bool SkipBom\n {\n get => skipBom;\n init => skipBom = checkPreamble = value;\n }\n\n public bool ConfigureAwait { get; init; } = false;\n\n public bool SyncRead { get; init; } = false;\n\n public Utf8StreamReader(Stream stream)\n : this(stream, DefaultBufferSize, false)\n {\n }\n\n public Utf8StreamReader(Stream stream, int bufferSize)\n : this(stream, bufferSize, false)\n {\n }\n\n public Utf8StreamReader(Stream stream, bool leaveOpen)\n : this(stream, DefaultBufferSize, leaveOpen)\n {\n }\n\n public Utf8StreamReader(Stream stream, int bufferSize, bool leaveOpen)\n {\n this.inputBuffer = ArrayPool.Shared.Rent(Math.Max(bufferSize, MinBufferSize));\n this.stream = stream;\n this.bufferSize = bufferSize;\n this.leaveOpen = leaveOpen;\n }\n\n public Utf8StreamReader(string path, FileOpenMode fileOpenMode = FileOpenMode.Throughput)\n : this(path, DefaultBufferSize, fileOpenMode)\n {\n }\n\n public Utf8StreamReader(string path, int bufferSize, FileOpenMode fileOpenMode = FileOpenMode.Throughput)\n : this(OpenPath(path, fileOpenMode), bufferSize, leaveOpen: false)\n {\n }\n\n static FileStream OpenPath(string path, FileOpenMode fileOpenMode = FileOpenMode.Throughput)\n {\n#if NETSTANDARD\n var useSequentialScan = FileOptions.SequentialScan;\n#else\n // SequentialScan is a perf hint that requires extra sys-call on non-Windows OSes.\n var useSequentialScan = OperatingSystem.IsWindows() ? FileOptions.SequentialScan : FileOptions.None;\n#endif\n var fileOptions = (fileOpenMode == FileOpenMode.Scalability)\n ? (FileOptions.Asynchronous | useSequentialScan)\n : useSequentialScan;\n return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, options: fileOptions);\n }\n\n#if !NETSTANDARD\n\n public Utf8StreamReader(string path, FileStreamOptions options)\n : this(path, options, DefaultBufferSize)\n {\n }\n\n public Utf8StreamReader(string path, FileStreamOptions options, int bufferSize)\n : this(OpenPath(path, options), bufferSize)\n {\n }\n\n static FileStream OpenPath(string path, FileStreamOptions options)\n {\n return new FileStream(path, options);\n }\n\n#endif\n\n // Peek() and EndOfStream is `Sync` method so does not provided.\n\n public Stream BaseStream => stream;\n\n public bool TryReadLine(out ReadOnlyMemory line)\n {\n ThrowIfDisposed();\n\n if (lastNewLinePosition >= 0)\n {\n line = inputBuffer.AsMemory(positionBegin, lastNewLinePosition - positionBegin);\n positionBegin = lastExaminedPosition + 1;\n lastNewLinePosition = lastExaminedPosition = -1;\n return true;\n }\n\n // AsSpan(positionBegin..positionEnd) is more readable but don't use range notation, it is slower.\n var index = IndexOfNewline(inputBuffer.AsSpan(positionBegin, positionEnd - positionBegin), out var newLineIndex);\n if (index == -1)\n {\n if (endOfStream && positionBegin != positionEnd)\n {\n // return last line\n line = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n positionBegin = positionEnd;\n return true;\n }\n\n lastNewLinePosition = lastExaminedPosition = -2; // not exists new line in this buffer\n line = default;\n return false;\n }\n\n line = inputBuffer.AsMemory(positionBegin, index); // positionBegin..(positionBegin+index)\n positionBegin = (positionBegin + newLineIndex + 1);\n lastNewLinePosition = lastExaminedPosition = -1;\n return true;\n }\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n public async ValueTask LoadIntoBufferAsync(CancellationToken cancellationToken = default)\n {\n ThrowIfDisposed();\n cancellationToken.ThrowIfCancellationRequested();\n\n // pre-check\n\n if (endOfStream)\n {\n if (positionBegin != positionEnd) // not yet fully consumed\n {\n return true;\n }\n else\n {\n return false;\n }\n }\n else\n {\n if (lastNewLinePosition >= 0) return true; // already filled line into buffer\n\n // lastNewLineIndex, lastExamined is relative from positionBegin\n if (lastNewLinePosition == -1)\n {\n var index = IndexOfNewline(inputBuffer.AsSpan(positionBegin, positionEnd - positionBegin), out var examinedIndex);\n if (index != -1)\n {\n // convert to absolute\n lastNewLinePosition = positionBegin + index;\n lastExaminedPosition = positionBegin + examinedIndex;\n return true;\n }\n }\n else\n {\n // change status to not searched\n lastNewLinePosition = -1;\n }\n }\n\n // requires load into buffer\n\n if (positionEnd != 0 && positionBegin == positionEnd)\n {\n // can reset buffer position\n positionBegin = positionEnd = 0;\n }\n\n var examined = positionEnd; // avoid to duplicate scan\n\n LOAD_INTO_BUFFER:\n // not reaches full, repeatedly read\n if (positionEnd != inputBuffer.Length)\n {\n var read = SyncRead\n ? stream.Read(inputBuffer.AsSpan(positionEnd))\n : await stream.ReadAsync(inputBuffer.AsMemory(positionEnd), cancellationToken).ConfigureAwait(ConfigureAwait);\n\n positionEnd += read;\n if (read == 0)\n {\n endOfStream = true;\n if (positionBegin != positionEnd) // has last line\n {\n return true;\n }\n else\n {\n return false;\n }\n }\n else\n {\n // first Read, require to check UTF8 BOM\n if (checkPreamble)\n {\n if (positionEnd < 3) goto LOAD_INTO_BUFFER;\n if (inputBuffer.AsSpan(0, 3).SequenceEqual(Encoding.UTF8.Preamble))\n {\n positionBegin = 3;\n }\n checkPreamble = false;\n }\n\n // scan examined(already scanned) to End.\n // Back one index to check if CRLF fell on buffer boundary\n var scanFrom = examined > 0 ? examined - 1 : examined;\n var index = IndexOfNewline(inputBuffer.AsSpan(scanFrom, positionEnd - scanFrom), out var examinedIndex);\n if (index != -1)\n {\n lastNewLinePosition = scanFrom + index;\n lastExaminedPosition = scanFrom + examinedIndex;\n return true;\n }\n\n examined = positionEnd;\n goto LOAD_INTO_BUFFER;\n }\n }\n\n // slide current buffer\n if (positionBegin != 0)\n {\n inputBuffer.AsSpan(positionBegin, positionEnd - positionBegin).CopyTo(inputBuffer);\n positionEnd -= positionBegin;\n positionBegin = 0;\n examined = positionEnd;\n goto LOAD_INTO_BUFFER;\n }\n\n // buffer is completely full, needs resize(positionBegin, positionEnd, examined are same)\n {\n var newBuffer = ArrayPool.Shared.Rent(GetNewSize(inputBuffer.Length));\n inputBuffer.AsSpan().CopyTo(newBuffer);\n ArrayPool.Shared.Return(inputBuffer);\n inputBuffer = newBuffer;\n goto LOAD_INTO_BUFFER;\n }\n }\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))]\n#endif\n public async ValueTask LoadIntoBufferAtLeastAsync(int minimumBytes, CancellationToken cancellationToken = default)\n {\n var loaded = positionEnd - positionBegin;\n if (minimumBytes < loaded)\n {\n return;\n }\n if (endOfStream)\n {\n throw new EndOfStreamException();\n }\n\n if (positionEnd != 0 && positionBegin == positionEnd)\n {\n // can reset buffer position\n loaded = positionBegin = positionEnd = 0;\n lastNewLinePosition = -1;\n }\n\n var remains = minimumBytes - loaded;\n\n if (inputBuffer.Length - positionEnd < remains)\n {\n // needs resize before load loop\n var newBuffer = ArrayPool.Shared.Rent(Math.Min(GetNewSize(inputBuffer.Length), positionEnd + remains));\n inputBuffer.AsSpan().CopyTo(newBuffer);\n ArrayPool.Shared.Return(inputBuffer);\n inputBuffer = newBuffer;\n }\n\n LOAD_INTO_BUFFER:\n var read = SyncRead\n ? stream.Read(inputBuffer.AsSpan(positionEnd))\n : await stream.ReadAsync(inputBuffer.AsMemory(positionEnd), cancellationToken).ConfigureAwait(ConfigureAwait);\n positionEnd += read;\n if (read == 0)\n {\n throw new EndOfStreamException();\n }\n else\n {\n // first Read, require to check UTF8 BOM\n if (checkPreamble)\n {\n if (positionEnd < 3) goto LOAD_INTO_BUFFER;\n if (inputBuffer.AsSpan(0, 3).SequenceEqual(Encoding.UTF8.Preamble))\n {\n positionBegin = 3;\n remains += 3; // read 3 bytes should not contains\n }\n checkPreamble = false;\n }\n\n remains -= read;\n if (remains < 0)\n {\n return;\n }\n\n goto LOAD_INTO_BUFFER;\n }\n }\n\n public async IAsyncEnumerable> ReadToEndChunksAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)\n {\n if (endOfStream)\n {\n var result = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n positionBegin = positionEnd;\n if (result.Length != 0)\n {\n yield return result;\n }\n yield break;\n }\n\n if (positionEnd != 0 && positionBegin != positionEnd)\n {\n yield return inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n }\n\n positionBegin = positionEnd = 0;\n lastNewLinePosition = -2;\n\n LOAD_INTO_BUFFER:\n var read = SyncRead\n ? stream.Read(inputBuffer.AsSpan(positionEnd))\n : await stream.ReadAsync(inputBuffer.AsMemory(positionEnd), cancellationToken).ConfigureAwait(ConfigureAwait);\n\n positionEnd += read;\n if (read == 0)\n {\n endOfStream = true;\n var result = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n positionBegin = positionEnd;\n if (result.Length != 0)\n {\n yield return result;\n }\n yield break;\n }\n else\n {\n // first Read, require to check UTF8 BOM\n if (checkPreamble)\n {\n if (positionEnd < 3) goto LOAD_INTO_BUFFER;\n if (inputBuffer.AsSpan(0, 3).SequenceEqual(Encoding.UTF8.Preamble))\n {\n positionBegin = 3;\n }\n checkPreamble = false;\n if (positionEnd - positionBegin == 0)\n {\n goto LOAD_INTO_BUFFER;\n }\n }\n\n yield return inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n positionBegin = positionEnd = 0;\n goto LOAD_INTO_BUFFER;\n }\n }\n\n public ValueTask ReadToEndAsync(CancellationToken cancellationToken = default)\n {\n return ReadToEndAsync(true, cancellationToken);\n }\n\n public ValueTask ReadToEndAsync(bool disableBomCheck, CancellationToken cancellationToken = default)\n {\n if (disableBomCheck && BaseStream is FileStream fs && fs.CanSeek)\n {\n return ReadToEndAsyncCore(fs.Length, true, cancellationToken);\n }\n\n return ReadToEndAsyncCore(-1, disableBomCheck, cancellationToken);\n }\n\n public ValueTask ReadToEndAsync(long resultSizeHint, CancellationToken cancellationToken = default)\n {\n return ReadToEndAsyncCore(resultSizeHint, true, cancellationToken);\n }\n\n async ValueTask ReadToEndAsyncCore(long resultSizeHint, bool disableBomCheck = true, CancellationToken cancellationToken = default)\n {\n if (endOfStream)\n {\n var slice = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n positionBegin = positionEnd = 0;\n lastNewLinePosition = -2;\n return (slice.Length != 0)\n ? slice.ToArray()\n : [];\n }\n\n if (resultSizeHint != -1)\n {\n if (resultSizeHint == 0)\n {\n return [];\n }\n\n var result = new byte[resultSizeHint];\n var memory = result.AsMemory();\n var totalRead = 0;\n\n if (positionEnd != 0 && positionBegin != positionEnd)\n {\n var slice = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n slice.CopyTo(memory);\n memory = memory.Slice(slice.Length);\n totalRead = slice.Length;\n }\n\n positionBegin = positionEnd = 0;\n lastNewLinePosition = -2;\n\n while (true)\n {\n var read = SyncRead\n ? stream.Read(memory.Span)\n : await stream.ReadAsync(memory, cancellationToken).ConfigureAwait(ConfigureAwait);\n totalRead += read;\n\n if (read == 0)\n {\n break;\n }\n else\n {\n memory = memory.Slice(read);\n if (memory.Length == 0)\n {\n // try to check stream is finished.\n var finalRead = SyncRead\n ? stream.Read(result.AsSpan(0, 1))\n : await stream.ReadAsync(result.AsMemory(0, 1), cancellationToken).ConfigureAwait(ConfigureAwait);\n\n if (finalRead == 0)\n {\n break;\n }\n else\n {\n throw new InvalidOperationException(\"resultSizeHint is smaller than data size.\");\n }\n }\n }\n }\n\n if (result.Length == totalRead)\n {\n return result;\n }\n else\n {\n return result.AsSpan(0, totalRead).ToArray();\n }\n }\n else\n {\n using var writer = new SegmentedArrayBufferWriter();\n if (positionEnd != 0 && positionBegin != positionEnd)\n {\n var slice = inputBuffer.AsMemory(positionBegin, positionEnd - positionBegin);\n writer.Write(slice.Span);\n }\n\n positionBegin = positionEnd = 0;\n lastNewLinePosition = -2;\n\n if (!disableBomCheck && checkPreamble && writer.WrittenCount == 0)\n {\n var memory = writer.GetMemory();\n var readCount = 0;\n READ_FOR_BOM:\n var read = SyncRead\n ? stream.Read(memory.Span)\n : await stream.ReadAsync(memory, cancellationToken).ConfigureAwait(ConfigureAwait);\n readCount += read;\n\n if (readCount < 3)\n {\n memory = memory.Slice(read);\n goto READ_FOR_BOM;\n }\n\n memory = writer.GetMemory();\n if (memory.Span.Slice(0, 3).SequenceEqual(Encoding.UTF8.Preamble))\n {\n // copy\n memory.Span.Slice(3).CopyTo(memory.Span);\n writer.Advance(readCount - 3);\n }\n else\n {\n writer.Advance(readCount);\n }\n\n checkPreamble = false;\n }\n\n while (true)\n {\n var read = SyncRead\n ? stream.Read(writer.GetMemory().Span)\n : await stream.ReadAsync(writer.GetMemory(), cancellationToken).ConfigureAwait(ConfigureAwait);\n\n if (read == 0)\n {\n break;\n }\n else\n {\n writer.Advance(read);\n }\n }\n\n endOfStream = true;\n return writer.ToArrayAndDispose();\n }\n }\n\n public ValueTask?> ReadLineAsync(CancellationToken cancellationToken = default)\n {\n if (TryReadLine(out var line))\n {\n return new ValueTask?>(line);\n }\n\n return Core(cancellationToken);\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n async ValueTask?> Core(CancellationToken cancellationToken)\n {\n if (await LoadIntoBufferAsync(cancellationToken).ConfigureAwait(ConfigureAwait))\n {\n if (TryReadLine(out var line))\n {\n return line;\n }\n }\n return null;\n }\n }\n\n public async IAsyncEnumerable> ReadAllLinesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)\n {\n while (await LoadIntoBufferAsync(cancellationToken).ConfigureAwait(ConfigureAwait))\n {\n while (TryReadLine(out var line))\n {\n yield return line;\n }\n }\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n public bool TryPeek(out byte data)\n {\n ThrowIfDisposed();\n\n if (positionEnd - positionBegin > 0)\n {\n data = inputBuffer[positionBegin];\n return true;\n }\n\n data = default;\n return false;\n }\n\n public ValueTask PeekAsync(CancellationToken cancellationToken = default)\n {\n if (TryPeek(out var data))\n {\n return new ValueTask(data);\n }\n\n return Core(cancellationToken);\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n async ValueTask Core(CancellationToken cancellationToken)\n {\n await LoadIntoBufferAtLeastAsync(1, cancellationToken);\n return inputBuffer[positionBegin];\n }\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n public bool TryRead(out byte data)\n {\n ThrowIfDisposed();\n\n if (TryPeek(out data))\n {\n positionBegin += 1;\n lastNewLinePosition = lastExaminedPosition = -1;\n return true;\n }\n\n data = default;\n return false;\n }\n\n public ValueTask ReadAsync(CancellationToken cancellationToken = default)\n {\n ThrowIfDisposed();\n\n if (TryRead(out var data))\n {\n return new ValueTask(data);\n }\n\n return Core(cancellationToken);\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n async ValueTask Core(CancellationToken cancellationToken)\n {\n await LoadIntoBufferAtLeastAsync(1, cancellationToken);\n TryRead(out var data);\n return data;\n }\n }\n\n public bool TryReadBlock(int count, out ReadOnlyMemory block)\n {\n ThrowIfDisposed();\n\n var loaded = positionEnd - positionBegin;\n if (count < loaded)\n {\n block = inputBuffer.AsMemory(positionBegin, count);\n positionBegin += count;\n lastNewLinePosition = lastExaminedPosition = -1;\n return true;\n }\n\n block = default;\n return false;\n }\n\n public ValueTask> ReadBlockAsync(int count, CancellationToken cancellationToken = default)\n {\n if (TryReadBlock(count, out var block))\n {\n return new ValueTask>(block);\n }\n\n return Core(count, cancellationToken);\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n async ValueTask> Core(int count, CancellationToken cancellationToken)\n {\n await LoadIntoBufferAtLeastAsync(count, cancellationToken);\n TryReadBlock(count, out var block);\n return block;\n }\n }\n\n static int GetNewSize(int capacity)\n {\n int newCapacity = unchecked(capacity * 2);\n if ((uint)newCapacity > ArrayMaxLength) newCapacity = ArrayMaxLength;\n return newCapacity;\n }\n\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\n static int IndexOfNewline(ReadOnlySpan span, out int examined)\n {\n // we only supports LF(\\n) or CRLF(\\r\\n).\n var indexOfNewLine = span.IndexOf((byte)'\\n');\n if (indexOfNewLine == -1)\n {\n examined = span.Length - 1;\n return -1;\n }\n examined = indexOfNewLine;\n\n if (indexOfNewLine >= 1 && span[indexOfNewLine - 1] == '\\r')\n {\n indexOfNewLine--; // case of '\\r\\n'\n }\n\n return indexOfNewLine;\n }\n\n // Reset API like Utf8JsonWriter\n\n public void Reset()\n {\n ThrowIfDisposed();\n ClearState();\n }\n\n public void Reset(Stream stream)\n {\n ThrowIfDisposed();\n ClearState();\n\n this.inputBuffer = ArrayPool.Shared.Rent(Math.Max(bufferSize, MinBufferSize));\n this.stream = stream;\n }\n\n public void Dispose()\n {\n if (isDisposed) return;\n\n isDisposed = true;\n ClearState();\n }\n\n public async ValueTask DisposeAsync()\n {\n if (isDisposed) return;\n\n isDisposed = true;\n if (!leaveOpen && stream != null)\n {\n await stream.DisposeAsync().ConfigureAwait(ConfigureAwait);\n stream = null!;\n }\n ClearState();\n }\n\n void ClearState()\n {\n if (inputBuffer != null)\n {\n ArrayPool.Shared.Return(inputBuffer);\n inputBuffer = null!;\n }\n\n if (!leaveOpen && stream != null)\n {\n stream.Dispose();\n stream = null!;\n }\n\n positionBegin = positionEnd = 0;\n endOfStream = false;\n checkPreamble = skipBom;\n lastNewLinePosition = lastExaminedPosition = -2;\n }\n\n void ThrowIfDisposed()\n {\n if (isDisposed) throw new ObjectDisposedException(\"\");\n }\n}\n"
},
{
"path": "src/Utf8StreamReader/Utf8StreamReader.csproj",
"content": "\n\n \n netstandard2.1;net6.0;net8.0\n 12\n enable\n enable\n Cysharp.IO\n\n true\n 1701;1702;1591;1573\n\n \n true\n string\n Utf8 based StreamReader for high performance text processing.\n \n\n \n \n \n\n \n \n \n\n \n \n all\n runtime; build; native; contentfiles; analyzers; buildtransitive\n \n \n\n\n"
},
{
"path": "src/Utf8StreamReader/Utf8TextReader.cs",
"content": "using System.Buffers;\nusing System.Runtime.CompilerServices;\nusing System.Text;\n\nnamespace Cysharp.IO;\n\npublic sealed class Utf8TextReader : IDisposable, IAsyncDisposable\n{\n const int DefaultCharBufferSize = 1024; // buffer per line.\n const int MinBufferSize = 128;\n\n readonly Utf8StreamReader reader;\n readonly int bufferSize;\n char[] outputBuffer;\n bool isDisposed;\n\n public Utf8TextReader(Utf8StreamReader reader)\n : this(reader, DefaultCharBufferSize)\n {\n }\n\n public Utf8TextReader(Utf8StreamReader reader, int bufferSize)\n {\n this.reader = reader;\n this.outputBuffer = ArrayPool.Shared.Rent(Math.Max(bufferSize, MinBufferSize));\n this.bufferSize = bufferSize;\n }\n\n public Stream BaseStream => reader.BaseStream;\n public Utf8StreamReader BaseReader => reader;\n\n public ValueTask LoadIntoBufferAsync(CancellationToken cancellationToken = default)\n {\n return reader.LoadIntoBufferAsync(cancellationToken);\n }\n\n public bool TryReadLine(out ReadOnlyMemory line)\n {\n if (!reader.TryReadLine(out var utf8Line))\n {\n line = default;\n return false;\n }\n\n var maxCharCount = Encoding.UTF8.GetMaxCharCount(utf8Line.Length);\n if (outputBuffer.Length < maxCharCount)\n {\n // need new buffer\n ArrayPool.Shared.Return(outputBuffer);\n outputBuffer = ArrayPool.Shared.Rent(maxCharCount);\n }\n\n var size = Encoding.UTF8.GetChars(utf8Line.Span, outputBuffer);\n line = outputBuffer.AsMemory(0, size);\n return true;\n }\n\n public ValueTask?> ReadLineAsync(CancellationToken cancellationToken = default)\n {\n if (TryReadLine(out var line))\n {\n return new ValueTask?>(line);\n }\n\n return Core(cancellationToken);\n\n#if !NETSTANDARD\n [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))]\n#endif\n async ValueTask?> Core(CancellationToken cancellationToken)\n {\n if (await LoadIntoBufferAsync(cancellationToken).ConfigureAwait(reader.ConfigureAwait))\n {\n if (TryReadLine(out var line))\n {\n return line;\n }\n }\n return null;\n }\n }\n\n public async IAsyncEnumerable> ReadAllLinesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default)\n {\n while (await LoadIntoBufferAsync(cancellationToken).ConfigureAwait(reader.ConfigureAwait))\n {\n while (TryReadLine(out var line))\n {\n yield return line;\n }\n }\n }\n \n // Utf8TextReader is a helper class for ReadOnlyMemory and string generation that internally holds Utf8StreamReader\n public async ValueTask ReadToEndAsync(CancellationToken cancellationToken = default)\n {\n // Using a method similar to .NET 9 LINQ to Objects's ToArray improvement, returns a structure optimized for gap-free sequential expansion\n // StreamReader.ReadToEnd copies the buffer to a StringBuilder, but this implementation holds char[] chunks(char[][]) without copying.\n using var writer = new SegmentedArrayBufferWriter();\n var decoder = Encoding.UTF8.GetDecoder();\n\n // Utf8StreamReader.ReadToEndChunksAsync returns the internal buffer ReadOnlyMemory as an asynchronous sequence upon each read completion\n await foreach (var chunk in reader.ReadToEndChunksAsync(cancellationToken).ConfigureAwait(reader.ConfigureAwait))\n {\n var input = chunk;\n while (input.Length != 0)\n {\n // The Decoder directly writes from the read buffer to the char[] buffer\n decoder.Convert(input.Span, writer.GetMemory().Span, flush: false, out var bytesUsed, out var charsUsed, out var completed);\n input = input.Slice(bytesUsed);\n writer.Advance(charsUsed);\n }\n }\n\n decoder.Convert([], writer.GetMemory().Span, flush: true, out _, out var finalCharsUsed, out _);\n writer.Advance(finalCharsUsed);\n\n // Directly generate a string from the char[][] buffer using String.Create\n return string.Create(writer.WrittenCount, writer, static (stringSpan, writer) =>\n {\n foreach (var item in writer.GetSegmentsAndDispose())\n {\n item.Span.CopyTo(stringSpan);\n stringSpan = stringSpan.Slice(item.Length);\n }\n });\n }\n\n public void Reset()\n {\n ThrowIfDisposed();\n ClearState();\n reader.Reset();\n }\n\n public void Reset(Stream stream)\n {\n ThrowIfDisposed();\n ClearState();\n\n outputBuffer = ArrayPool.Shared.Rent(Math.Max(bufferSize, MinBufferSize));\n reader.Reset(stream);\n }\n\n public void Dispose()\n {\n if (isDisposed) return;\n\n isDisposed = true;\n ClearState();\n reader.Dispose();\n }\n\n public ValueTask DisposeAsync()\n {\n if (isDisposed) return default;\n\n isDisposed = true;\n ClearState();\n return reader.DisposeAsync();\n }\n\n void ClearState()\n {\n if (outputBuffer != null)\n {\n ArrayPool.Shared.Return(outputBuffer);\n outputBuffer = null!;\n }\n }\n\n void ThrowIfDisposed()\n {\n if (isDisposed) throw new ObjectDisposedException(\"\");\n }\n}\n\npublic static class Utf8StreamReaderExtensions\n{\n public static Utf8TextReader AsTextReader(this Utf8StreamReader reader) => new Utf8TextReader(reader);\n public static Utf8TextReader AsTextReader(this Utf8StreamReader reader, int bufferSize) => new Utf8TextReader(reader, bufferSize);\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/FakeMemoryStream.cs",
"content": "#pragma warning disable CS1998\n\nusing System.Runtime.CompilerServices;\nusing System.Runtime.InteropServices;\n\nnamespace Utf8StreamReaderTests;\n\ninternal class FakeMemoryStream : Stream\n{\n #region NotImplemented\n\n public override bool CanRead => true;\n\n public override bool CanSeek => throw new NotImplementedException();\n\n public override bool CanWrite => throw new NotImplementedException();\n\n public override long Length => throw new NotImplementedException();\n\n public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }\n\n public override void Flush()\n {\n throw new NotImplementedException();\n }\n\n public override int Read(byte[] buffer, int offset, int count)\n {\n throw new NotImplementedException();\n }\n\n public override long Seek(long offset, SeekOrigin origin)\n {\n throw new NotImplementedException();\n }\n\n public override void SetLength(long value)\n {\n throw new NotImplementedException();\n }\n\n public override void Write(byte[] buffer, int offset, int count)\n {\n throw new NotImplementedException();\n }\n\n #endregion\n\n public bool IsDisposed { get; set; }\n\n protected override void Dispose(bool disposing)\n {\n IsDisposed = true;\n }\n\n Memory[] lastAddedData = default!;\n Queue>> data = new();\n\n public void AddMemory(params Memory[] memories)\n {\n foreach (Memory mem in memories)\n {\n if (mem.Length == 0) throw new ArgumentException(\"Length 0 is not allowed.\");\n data.Enqueue(new(mem));\n }\n this.lastAddedData = memories;\n }\n\n public void Restart()\n {\n data.Clear();\n AddMemory(lastAddedData);\n }\n\n public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default)\n {\n if (data.Count == 0)\n {\n return 0;\n }\n\n var memory = data.Peek().Value;\n\n var copySize = Math.Min(memory.Length, buffer.Length);\n memory.Slice(0, copySize).CopyTo(buffer);\n var newMemory = memory.Slice(copySize);\n if (newMemory.Length == 0)\n {\n data.Dequeue();\n }\n else\n {\n data.Peek().Value = newMemory;\n }\n\n return copySize;\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/FileReadTest.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Reflection;\nusing System.Text;\nusing System.Threading.Tasks;\n\nnamespace Utf8StreamReaderTests;\n\npublic class FileReadTest(ITestOutputHelper Console)\n{\n [Fact]\n public async Task ReadPath()\n {\n var path1 = Path.Combine(Path.GetDirectoryName(typeof(FileReadTest).Assembly.FullName!)!, \"file1.txt\");\n var actual = await Utf8StreamReaderResultAsync(path1);\n\n actual.Should().Equal([\n \"abcde\",\n \"fgh\",\n \"ijklmnopqrs\"\n ]);\n }\n\n static async Task Utf8StreamReaderResultAsync(string path)\n {\n using var reader = new Utf8StreamReader(path);\n var l = new List();\n await foreach (var item in reader.ReadAllLinesAsync())\n {\n l.Add(Encoding.UTF8.GetString(item.Span));\n }\n return l.ToArray();\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/ReadBlockTest.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Threading.Tasks;\n\nnamespace Utf8StreamReaderTests;\n\npublic class ReadBlockTest\n{\n [Fact]\n public async Task LineAndBlock()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n //var sr = new StreamReader(ms);\n //var a = await sr.ReadLineAsync();\n //var buf = new char[1024];\n //await sr.ReadBlockAsync(buf.AsMemory(0, 10));\n\n\n var reader = new Utf8StreamReader(ms);\n ToString((await reader.ReadLineAsync()).Value).Should().Be(\"abc\");\n\n ToString((await reader.ReadBlockAsync(2))).Should().Be(\"de\");\n\n ToString((await reader.ReadLineAsync()).Value).Should().Be(\"f\");\n\n ToString((await reader.ReadBlockAsync(8))).Should().Be(\"ghij\\nzkl\");\n\n ToString((await reader.ReadLineAsync()).Value).Should().Be(\"mno\");\n\n\n }\n\n static byte[] GetBytes(string x)\n {\n return Encoding.UTF8.GetBytes(x);\n }\n\n static string ToString(ReadOnlyMemory buffer)\n {\n return Encoding.UTF8.GetString(buffer.Span);\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/ReadTest.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Threading.Tasks;\n\nnamespace Utf8StreamReaderTests;\n\npublic class ReadTest\n{\n [Fact]\n public async Task ReadToEndAsync()\n {\n // with bom\n {\n var bom = Encoding.UTF8.GetPreamble();\n\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n new byte[] { bom[0] },\n new byte[] { bom[1] },\n new byte[] { bom[2], (byte)'Z' },\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var sr = new Utf8StreamReader(ms);\n var result = await sr.ReadToEndAsync(disableBomCheck: false);\n\n var expected = \"Zabc\\ndef\\r\\nghij\\nzklmno\\r\\n\\n\";\n var actual = ToString(result);\n\n actual.Should().Be(expected);\n }\n // no bom\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n new byte[] { (byte)'Z' },\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var sr = new Utf8StreamReader(ms);\n var result = await sr.ReadToEndAsync();\n\n var expected = \"Zabc\\ndef\\r\\nghij\\nzklmno\\r\\n\\n\";\n var actual = ToString(result);\n\n actual.Should().Be(expected);\n }\n }\n\n [Fact]\n public async Task ReadToEndChunks()\n {\n var bom = Encoding.UTF8.GetPreamble();\n\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n new byte[] { bom[0] },\n new byte[] { bom[1] },\n new byte[] { bom[2], (byte)'Z' },\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var sr = new Utf8StreamReader(ms);\n\n var list = new List();\n await foreach (var item in sr.ReadToEndChunksAsync())\n {\n list.Add(item.ToArray());\n }\n\n ToString(list[0]).Should().Be(\"Z\");\n ToString(list[1]).Should().Be(\"a\");\n ToString(list[2]).Should().Be(\"bc\\n\");\n ToString(list[3]).Should().Be(\"def\\r\\n\");\n ToString(list[4]).Should().Be(\"ghij\\n\");\n ToString(list[5]).Should().Be(\"zklmno\\r\\n\\n\");\n }\n\n [Fact]\n public async Task TestPeek()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var sr = new Utf8StreamReader(ms);\n\n sr.TryPeek(out var data).Should().BeFalse();\n (await sr.PeekAsync()).Should().Be((byte)'a');\n sr.TryPeek(out data).Should().BeTrue();\n data.Should().Be((byte)'a');\n\n ToString(await sr.ReadLineAsync()).Should().Be(\"abc\");\n\n (await sr.PeekAsync()).Should().Be((byte)'d');\n\n ToString(await sr.ReadLineAsync()).Should().Be(\"def\");\n }\n\n // LoadIntoBufferAtLeastAsync\n // TryRead\n // ReadAsync\n\n [Fact]\n public async Task TestRead()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var sr = new Utf8StreamReader(ms);\n\n await sr.LoadIntoBufferAtLeastAsync(2);\n\n sr.TryRead(out var a).Should().BeTrue();\n a.Should().Be((byte)'a');\n\n sr.TryRead(out var b).Should().BeTrue();\n b.Should().Be((byte)'b');\n\n sr.TryRead(out var c).Should().BeTrue();\n c.Should().Be((byte)'c');\n\n sr.TryRead(out var n).Should().BeTrue();\n n.Should().Be((byte)'\\n');\n\n sr.TryRead(out _).Should().BeFalse();\n\n (await sr.ReadAsync()).Should().Be((byte)'d');\n }\n\n static byte[] GetBytes(string x)\n {\n return Encoding.UTF8.GetBytes(x);\n }\n\n static string ToString(ReadOnlyMemory? buffer)\n {\n if (buffer == null) return null!;\n return Encoding.UTF8.GetString(buffer.Value.Span);\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/ReadToEndTest.cs",
"content": "using System.Text;\n\nnamespace Utf8StreamReaderTests;\n\npublic class ReadToEndTest\n{\n\n [Fact]\n public async Task AfterRead()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var all = await new Utf8StreamReader(ms).ReadToEndAsync();\n\n ms.Restart();\n\n var reader = new Utf8StreamReader(ms);\n await reader.ReadLineAsync();\n\n var expected = \"def\\r\\nghij\\nzklmno\\r\\n\\n\";\n\n var actual = await reader.ReadToEndAsync(resultSizeHint: all.Length);\n\n Encoding.UTF8.GetString(actual).Should().Be(expected);\n }\n\n [Fact]\n public async Task SmallHint()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var reader = new Utf8StreamReader(ms);\n\n var expected = \"abc\\ndef\\r\\nghij\\nzklmno\\r\\n\\n\";\n\n await Assert.ThrowsAsync(async () =>\n {\n var actual = await reader.ReadToEndAsync(resultSizeHint: Encoding.UTF8.GetByteCount(expected) - 2);\n });\n }\n\n [Fact]\n public async Task Just()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"zklmno\\r\\n\\n\"));\n\n var reader = new Utf8StreamReader(ms);\n\n var expected = \"abc\\ndef\\r\\nghij\\nzklmno\\r\\n\\n\";\n\n var actual = await reader.ReadToEndAsync(resultSizeHint: expected.Length);\n Encoding.UTF8.GetString(actual).Should().Be(expected);\n }\n\n static byte[] GetBytes(string x)\n {\n return Encoding.UTF8.GetBytes(x);\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/SegmentedArrayBufferWriterTest.cs",
"content": "namespace Utf8StreamReaderTests;\n\npublic class SegmentedArrayBufferWriterTest\n{\n [Fact(Skip = \"Reduce memory usage in CI\")]\n public void AllocateFull()\n {\n var writer = new SegmentedArrayBufferWriter();\n\n var memCount = 8192;\n long total = 0;\n for (int i = 0; i < 18; i++)\n {\n var mem = writer.GetMemory();\n mem.Length.Should().Be(memCount);\n total += mem.Length;\n memCount *= 2;\n writer.Advance(mem.Length);\n }\n\n Memory lastMemory = writer.GetMemory();\n (total).Should().BeLessThan(Array.MaxLength);\n (total + lastMemory.Length).Should().BeGreaterThan(Array.MaxLength);\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/Tests.cs",
"content": "using System.Buffers;\nusing System.Text;\n\nnamespace Utf8StreamReaderTests;\n\npublic class Tests(ITestOutputHelper Console)\n{\n [Fact]\n public async Task Standard()\n {\n var originalStrings = \"\"\"\nfoo\nbare\n\nbaz boz too\n\"\"\";\n\n var stream = CreateStringStream(originalStrings);\n\n\n using var reader = new Utf8StreamReader(stream);\n\n var sb = new StringBuilder();\n\n bool isFirst = true;\n ReadOnlyMemory? line;\n while ((line = await reader.ReadLineAsync()) != null)\n {\n if (isFirst) isFirst = false;\n else sb.AppendLine();\n\n Console.WriteLine(Encoding.UTF8.GetString(line.Value.Span));\n sb.Append(Encoding.UTF8.GetString(line.Value.Span));\n }\n\n sb.ToString().Should().Be(originalStrings.ToString());\n }\n\n [Fact]\n public async Task BOM()\n {\n var bytes = Encoding.UTF8.GetPreamble().Concat(\"\"\"\nfoo\nbare\n\nbaz boz too\n\"\"\"u8.ToArray()).ToArray();\n\n var bomStrings = Encoding.UTF8.GetString(bytes);\n\n var stream = CreateStringStream(bomStrings);\n\n var originalStrings = \"\"\"\nfoo\nbare\n\nbaz boz too\n\"\"\";\n\n using var reader = new Utf8StreamReader(stream);\n\n var sb = new StringBuilder();\n\n bool isFirst = true;\n ReadOnlyMemory? line;\n while ((line = await reader.ReadLineAsync()) != null)\n {\n if (isFirst) isFirst = false;\n else sb.AppendLine();\n\n Console.WriteLine(Encoding.UTF8.GetString(line.Value.Span));\n sb.Append(Encoding.UTF8.GetString(line.Value.Span));\n }\n\n sb.ToString().Should().Be(originalStrings.ToString());\n }\n\n [Fact]\n public async Task NewLineCheck()\n {\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"jklmno\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"jklmno\\r\\n\")); // + last new line\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"a\"),\n GetBytes(\"bc\\n\"),\n GetBytes(\"def\\r\\n\"),\n GetBytes(\"ghij\\n\"),\n GetBytes(\"jklmno\\r\\n\\n\")); // + last new line x2\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n }\n\n [Fact]\n public async Task BOM2()\n {\n {\n var ms = new FakeMemoryStream();\n\n // small bom\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n GetBytes(\"a\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n {\n var ms = new FakeMemoryStream();\n\n // long bom\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n GetBytes(\"abcdefghijklmnopqrastu\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n\n // yes bom\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n GetBytes(\"あいうえお\")); // japanese hiragana.\n\n var reader = new Utf8StreamReader(ms) { SkipBom = false };\n var line = await reader.ReadLineAsync();\n line.Value.Slice(0, 3).Span.SequenceEqual(Encoding.UTF8.Preamble).Should().BeTrue();\n line.Value.Slice(3).Span.SequenceEqual(GetBytes(\"あいうえお\")).Should().BeTrue();\n }\n }\n\n [Fact]\n public async Task EmptyString()\n {\n {\n var ms = new MemoryStream();\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms = new MemoryStream();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n // bom only\n {\n var ms = new FakeMemoryStream();\n\n // small bom\n ms.AddMemory(Encoding.UTF8.GetPreamble());\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n // newline only\n {\n var ms = new FakeMemoryStream();\n\n // small bom\n ms.AddMemory(GetBytes(\"\\r\\n\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n // newline only 2\n {\n var ms = new FakeMemoryStream();\n\n // small bom\n ms.AddMemory(GetBytes(\"\\n\\r\\n\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n }\n\n [Fact]\n public async Task SmallString()\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(GetBytes(\"z\"));\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n\n // minbuffer = 1024\n\n [Fact]\n public async Task Resize()\n {\n var bufferSize = 1024;\n\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"!!!\\r\\n\"), // first line consume\n GetBytes(new string('a', 1018)),\n GetBytes(\"bcdefghijklmnopqrstuvwxyz\\r\\n\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms, bufferSize);\n\n actual[1].Should().Be(expected[1]);\n actual.Should().Equal(expected);\n }\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(\"!!!\\r\\n\"), // first line consume\n GetBytes(new string('a', 1018)),\n GetBytes(\"bcdefghijklmnopqrstuvwxyz\\r\\n\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms, bufferSize);\n\n actual[1].Should().Be(expected[1]);\n actual.Should().Equal(expected);\n }\n }\n\n [Fact]\n public async Task OnlySlice()\n {\n var bufferSize = 1024;\n\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(new string('a', 1018) + \"\\r\\nbcdefgh\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms, bufferSize);\n\n actual[1].Should().Be(expected[1]);\n actual.Should().Equal(expected);\n }\n }\n\n [Fact]\n public async Task HugeBuffer()\n {\n var bufferSize = 1024;\n\n {\n var ms = new FakeMemoryStream();\n\n ms.AddMemory(\n GetBytes(new string('a', 30000) + \"\\r\\nb\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8StreamReaderResultAsync(ms, bufferSize);\n\n actual[1].Should().Be(expected[1]);\n actual.Should().Equal(expected);\n }\n }\n\n [Fact]\n public async Task NewLineTrimmedAtBufferBoundary()\n {\n // Buffer 1: aaa....\\r\\nasdf\\r\n // Buffer 2: \\nasdf\n var ms2 = CreateStringStream(\n new string('a', 1017) +\n \"\\r\\nasdf\" +\n \"\\r\\nasdf\");\n\n var actual = await Utf8StreamReaderResultAsync(ms2, 1024);\n\n string[] expected =\n [\n new string('a', 1017),\n \"asdf\",\n \"asdf\",\n ];\n\n actual[1].Should().Be(expected[1]);\n actual.Should().Equal(expected);\n }\n\n static async Task Utf8StreamReaderResultAsync(Stream ms, int? size = null)\n {\n var reader = (size == null) ? new Utf8StreamReader(ms) : new Utf8StreamReader(ms, size.Value);\n var l = new List();\n await foreach (var item in reader.ReadAllLinesAsync())\n {\n l.Add(GetString(item));\n }\n return l.ToArray();\n }\n\n static async Task StreamReaderResultAsync(Stream ms)\n {\n var reader = new StreamReader(ms);\n var l = new List();\n string? s;\n while ((s = (await reader.ReadLineAsync())) != null)\n {\n l.Add(s);\n }\n return l.ToArray();\n }\n\n static string GetString(ReadOnlyMemory x)\n {\n return Encoding.UTF8.GetString(x.Span);\n }\n\n static byte[] GetBytes(string x)\n {\n return Encoding.UTF8.GetBytes(x);\n }\n\n static MemoryStream CreateStringStream(string input) => new(Encoding.UTF8.GetBytes(input));\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/TextReaderTest.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Threading.Tasks;\n\nnamespace Utf8StreamReaderTests;\n\npublic class TextReaderTest\n{\n [Fact]\n public async Task ReadLine()\n {\n var ms = new FakeMemoryStream();\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n GetBytes(new string('a', 30000) + \"\\r\\nb\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n var expected = await StreamReaderResultAsync(ms);\n\n ms.Restart();\n\n var actual = await Utf8TextReaderResultAsync(ms);\n\n actual.Should().Equal(expected);\n }\n\n [Fact]\n public async Task ReadToEnd()\n {\n var ms = new FakeMemoryStream();\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n GetBytes(new string('a', 30000) + \"\\r\\nb\"),\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n using var sr = new StreamReader(ms, leaveOpen: true);\n var expected = await sr.ReadToEndAsync();\n\n ms.Restart();\n\n using var usr = new Utf8StreamReader(ms).AsTextReader();\n var actual = await usr.ReadToEndAsync();\n\n actual.Should().Be(expected);\n }\n\n [Fact]\n public async Task ReadToEndLeftOver()\n {\n var ms = new FakeMemoryStream();\n\n var hiragana = Encoding.UTF8.GetBytes(\"あ\"); // 3 byte\n\n ms.AddMemory(\n Encoding.UTF8.GetPreamble(),\n new byte[] { hiragana[0] },\n new byte[] { hiragana[1] },\n new byte[] { hiragana[2] },\n GetBytes(\"あいうえおかきくけこ\\n\"),\n GetBytes(\"ABCDEFGHIJKLMN\")\n );\n\n using var sr = new StreamReader(ms, leaveOpen: true);\n var expected = await sr.ReadToEndAsync();\n\n ms.Restart();\n\n using var usr = new Utf8StreamReader(ms).AsTextReader();\n var actual = await usr.ReadToEndAsync();\n\n actual.Should().Be(expected);\n }\n\n static async Task Utf8TextReaderResultAsync(Stream ms)\n {\n using var reader = new Utf8StreamReader(ms).AsTextReader();\n var l = new List();\n await foreach (var item in reader.ReadAllLinesAsync())\n {\n l.Add(item.ToString());\n }\n return l.ToArray();\n }\n\n static async Task StreamReaderResultAsync(Stream ms)\n {\n var reader = new StreamReader(ms);\n var l = new List();\n string? s;\n while ((s = (await reader.ReadLineAsync())) != null)\n {\n l.Add(s);\n }\n return l.ToArray();\n }\n static string GetString(ReadOnlyMemory x)\n {\n return Encoding.UTF8.GetString(x.Span);\n }\n\n static byte[] GetBytes(string x)\n {\n return Encoding.UTF8.GetBytes(x);\n }\n}\n"
},
{
"path": "tests/Utf8StreamReader.Tests/Utf8StreamReader.Tests.csproj",
"content": "\n\n \n net8.0\n enable\n enable\n Utf8StreamReaderTests\n true\n 9113\n \n\n \n \n \n all\n runtime; build; native; contentfiles; analyzers; buildtransitive\n \n \n \n \n runtime; build; native; contentfiles; analyzers; buildtransitive\n all\n \n \n\n \n \n \n\n \n \n \n \n \n \n\n \n \n Always\n \n \n\n\n"
},
{
"path": "tests/Utf8StreamReader.Tests/file1.txt",
"content": "abcde\nfgh\nijklmnopqrs\n"
}
]