Repository: Octonica/ClickHouseClient Branch: master Commit: bdd073c40dea Files: 265 Total size: 1.9 MB Directory structure: gitextract_vv1_tqul/ ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── docs/ │ ├── ClickHouseColumnWriter.md │ ├── Parameters.md │ └── TypeMapping.md └── src/ ├── ConnectionSettingsHelper.cs ├── Octonica.ClickHouseClient/ │ ├── ClickHouseBinaryProtocolReader.cs │ ├── ClickHouseBinaryProtocolWriter.cs │ ├── ClickHouseColumnSettings.cs │ ├── ClickHouseColumnWriter.cs │ ├── ClickHouseCommand.cs │ ├── ClickHouseConnection.cs │ ├── ClickHouseConnectionSettings.cs │ ├── ClickHouseConnectionState.cs │ ├── ClickHouseConnectionStringBuilder.cs │ ├── ClickHouseDataReader.cs │ ├── ClickHouseDataReaderBase.cs │ ├── ClickHouseDataReaderRowLimit.cs │ ├── ClickHouseDataReaderState.cs │ ├── ClickHouseDbProviderFactory.cs │ ├── ClickHouseDbType.cs │ ├── ClickHouseFlushMode.cs │ ├── ClickHouseParameter.cs │ ├── ClickHouseParameterCollection.cs │ ├── ClickHouseParameterMode.cs │ ├── ClickHousePasswordComplexityRule.cs │ ├── ClickHouseQueryExecutionProgress.cs │ ├── ClickHouseReaderColumnSettings.cs │ ├── ClickHouseServerInfo.cs │ ├── ClickHouseTable.cs │ ├── ClickHouseTableColumn.cs │ ├── ClickHouseTableColumnCollection.cs │ ├── ClickHouseTableProvider.cs │ ├── ClickHouseTableProviderCollection.cs │ ├── ClickHouseTableWriter.cs │ ├── ClickHouseTcpClient.cs │ ├── ClickHouseTcpClientState.cs │ ├── ClickHouseTlsMode.cs │ ├── ClickHouseVersion.cs │ ├── Exceptions/ │ │ ├── ClickHouseErrorCodes.cs │ │ ├── ClickHouseException.cs │ │ ├── ClickHouseHandledException.cs │ │ └── ClickHouseServerException.cs │ ├── IClickHouseArrayTableColumn.cs │ ├── IClickHouseColumnDescriptor.cs │ ├── IClickHouseSessionExternalResources.cs │ ├── IClickHouseTableColumn.cs │ ├── IClickHouseTableColumnDispatcher.cs │ ├── IClickHouseTableProvider.cs │ ├── Octonica.ClickHouseClient.csproj │ ├── Properties/ │ │ └── AssemblyInfo.cs │ ├── Protocol/ │ │ ├── BlockFieldCodes.cs │ │ ├── BlockHeader.cs │ │ ├── CityHash.cs │ │ ├── ClickHouseEmptyTableWriter.cs │ │ ├── ClickHouseParameterWriter.cs │ │ ├── ClickHouseProtocolRevisions.cs │ │ ├── ClickHouseSyntaxHelper.cs │ │ ├── ClientHelloMessage.cs │ │ ├── ClientMessageCode.cs │ │ ├── ClientQueryMessage.cs │ │ ├── ColumnInfo.cs │ │ ├── CompressionAlgorithm.cs │ │ ├── CompressionDecoderBase.cs │ │ ├── CompressionEncoderBase.cs │ │ ├── IClickHouseColumnReader.cs │ │ ├── IClickHouseColumnReaderBase.cs │ │ ├── IClickHouseColumnWriter.cs │ │ ├── IClickHouseColumnWriterFactory.cs │ │ ├── IClickHouseParameterValueWriter.cs │ │ ├── IClickHouseParameterWriter.cs │ │ ├── IClickHouseTableWriter.cs │ │ ├── IClientMessage.cs │ │ ├── IServerMessage.cs │ │ ├── Lz4CompressionDecoder.cs │ │ ├── Lz4CompressionEncoder.cs │ │ ├── QueryKind.cs │ │ ├── SequenceSize.cs │ │ ├── ServerDataMessage.cs │ │ ├── ServerEndOfStreamMessage.cs │ │ ├── ServerErrorMessage.cs │ │ ├── ServerHelloMessage.cs │ │ ├── ServerMessageCode.cs │ │ ├── ServerPongMessage.cs │ │ ├── ServerProfileInfoMessage.cs │ │ ├── ServerProgressMessage.cs │ │ ├── ServerTableColumnsMessage.cs │ │ ├── ServerTimeZoneUpdateMessage.cs │ │ ├── StateCodes.cs │ │ └── UnknownServerMessage.cs │ ├── Types/ │ │ ├── ArrayTableColumn.cs │ │ ├── ArrayTypeInfo.cs │ │ ├── BigIntegerTableColumn.cs │ │ ├── BigIntegerTypeInfoBase.cs │ │ ├── BoolTableColumn.cs │ │ ├── BoolTypeInfo.cs │ │ ├── ClickHouseColumnReinterpreter.cs │ │ ├── ClickHouseColumnSerializationMode.cs │ │ ├── ClickHouseEnumConverter.cs │ │ ├── ClickHouseTableColumnHelper.cs │ │ ├── ClickHouseTypeInfoProvider.cs │ │ ├── CustomSerializationColumnReader.cs │ │ ├── CustomSerializationSkippingColumnReader.cs │ │ ├── Date32TableColumn.Net6.0.cs │ │ ├── Date32TableColumn.NetCoreApp3.1.cs │ │ ├── Date32TableColumn.cs │ │ ├── Date32TypeInfo.Net6.0.cs │ │ ├── Date32TypeInfo.NetCoreApp3.1.cs │ │ ├── Date32TypeInfo.cs │ │ ├── DateOnlyTableColumn.cs │ │ ├── DateTime64TableColumn.cs │ │ ├── DateTime64TypeInfo.cs │ │ ├── DateTimeTableColumn.cs │ │ ├── DateTimeTypeInfo.cs │ │ ├── DateTypeInfo.Net6.0.cs │ │ ├── DateTypeInfo.NetCoreApp3.1.cs │ │ ├── DateTypeInfo.cs │ │ ├── Decimal128TypeInfo.cs │ │ ├── Decimal32TypeInfo.cs │ │ ├── Decimal64TypeInfo.cs │ │ ├── DecimalTableColumn.cs │ │ ├── DecimalTypeInfo.cs │ │ ├── DecimalTypeInfoBase.cs │ │ ├── EmptyParameterValueWriter.cs │ │ ├── Enum16TypeInfo.cs │ │ ├── Enum8TypeInfo.cs │ │ ├── EnumTableColumn.cs │ │ ├── EnumTypeInfoBase.cs │ │ ├── FixedStringDecodedCharArrayTableColumn.cs │ │ ├── FixedStringDecodedTableColumn.cs │ │ ├── FixedStringTableColumn.cs │ │ ├── FixedStringTableColumnBase.cs │ │ ├── FixedStringTypeInfo.cs │ │ ├── Float32TableColumn.cs │ │ ├── Float32TypeInfo.cs │ │ ├── Float64TypeInfo.cs │ │ ├── HexStringLiteralValueWriter.cs │ │ ├── HexStringLiteralWriterCastMode.cs │ │ ├── HexStringParameterWriter.cs │ │ ├── IClickHouseColumnReinterpreter.cs │ │ ├── IClickHouseColumnTypeDescriptor.cs │ │ ├── IClickHouseColumnTypeInfo.cs │ │ ├── IClickHouseConfigurableTypeInfo.cs │ │ ├── IClickHouseEnumConverter.cs │ │ ├── IClickHouseReinterpretedTableColumn.cs │ │ ├── IClickHouseTypeInfo.cs │ │ ├── IClickHouseTypeInfoProvider.cs │ │ ├── Int128TypeInfo.cs │ │ ├── Int16TableColumn.cs │ │ ├── Int16TypeInfo.cs │ │ ├── Int256TypeInfo.cs │ │ ├── Int32TableColumn.cs │ │ ├── Int32TypeInfo.cs │ │ ├── Int64TypeInfo.cs │ │ ├── Int8TableColumn.cs │ │ ├── Int8TypeInfo.cs │ │ ├── IntermediateClickHouseTypeInfo.cs │ │ ├── IpColumnReaderBase.cs │ │ ├── IpV4TableColumn.cs │ │ ├── IpV4TypeInfo.cs │ │ ├── IpV6TableColumn.cs │ │ ├── IpV6TypeInfo.cs │ │ ├── KeyValuePairTableColumn.cs │ │ ├── LowCardinalityTableColumn.cs │ │ ├── LowCardinalityTypeInfo.cs │ │ ├── MapTypeInfo.cs │ │ ├── NothingTableColumn.cs │ │ ├── NothingTypeInfo.cs │ │ ├── NullableTableColumn.cs │ │ ├── NullableTypeInfo.cs │ │ ├── ObjectColumnAdapter.cs │ │ ├── ReinterpretedArrayTableColumn.cs │ │ ├── ReinterpretedObjectTableColumn.cs │ │ ├── ReinterpretedTableColumn.cs │ │ ├── SimpleLiteralValueWriter.cs │ │ ├── SimpleParameterWriter.cs │ │ ├── SimpleSkippingColumnReader.cs │ │ ├── SimpleTypeInfo.cs │ │ ├── SparseColumn.cs │ │ ├── StringByteArrayTableColumn.cs │ │ ├── StringCharArrayTableColumn.cs │ │ ├── StringLiteralValueWriter.cs │ │ ├── StringParameterWriter.cs │ │ ├── StringTableColumn.cs │ │ ├── StringTableColumnBase.cs │ │ ├── StringTypeInfo.cs │ │ ├── StructureReaderBase.cs │ │ ├── StructureTableColumn.cs │ │ ├── StructureWriterBase.cs │ │ ├── TupleTableColumn.cs │ │ ├── TupleTypeInfo.cs │ │ ├── UInt128TypeInfo.cs │ │ ├── UInt16TableColumn.cs │ │ ├── UInt16TypeInfo.cs │ │ ├── UInt256TypeInfo.cs │ │ ├── UInt32TableColumn.cs │ │ ├── UInt32TypeInfo.cs │ │ ├── UInt64TypeInfo.cs │ │ ├── UInt8TableColumn.cs │ │ ├── UInt8TypeInfo.cs │ │ ├── UuidTypeInfo.cs │ │ ├── VariantTableColumn.cs │ │ └── VariantTypeInfo.cs │ └── Utils/ │ ├── CertificateHelper.Net5.0.cs │ ├── CertificateHelper.NetCoreApp3.1.cs │ ├── CertificateHelper.cs │ ├── CommonUtils.cs │ ├── ConstantReadOnlyList.cs │ ├── FunctionHelper.cs │ ├── ICollectionList.cs │ ├── IConverter.cs │ ├── IConverterDispatcher.cs │ ├── IReadOnlyListExt.cs │ ├── ITypeDispatcher.cs │ ├── IndexedCollectionBase.cs │ ├── ListExtensions.cs │ ├── ListSpan.cs │ ├── MappedListSpan.cs │ ├── MappedReadOnlyList.cs │ ├── MappedReadOnlyListSpan.cs │ ├── MemoryCollectionList.cs │ ├── MultiDimensionalArrayReadOnlyListAdapter.cs │ ├── ReadOnlyCollectionList.cs │ ├── ReadOnlyListSpan.cs │ ├── ReadOnlyMemoryCollectionList.cs │ ├── ReadOnlyMemoryList.cs │ ├── ReadWriteBuffer.cs │ ├── SimpleReadOnlySequenceSegment.cs │ ├── TaskHelper.cs │ ├── TimeZoneHelper.Net6.0.cs │ ├── TimeZoneHelper.NetCoreApp3.1.cs │ ├── TimeZoneHelper.cs │ └── TypeDispatcher.cs ├── Octonica.ClickHouseClient.Benchmarks/ │ ├── ColumnWriterBenchmarks.cs │ ├── InsertRowBenchmark.cs │ ├── Octonica.ClickHouseClient.Benchmarks.csproj │ └── Program.cs ├── Octonica.ClickHouseClient.Tests/ │ ├── AsyncTestFibSequence.cs │ ├── CityHashTests.Data.cs │ ├── CityHashTests.cs │ ├── ClickHouseColumnWriterTests.cs │ ├── ClickHouseCommandTests.cs │ ├── ClickHouseConnectionStringBuilderTests.cs │ ├── ClickHouseConnectionTests.cs │ ├── ClickHouseDataReaderTests.cs │ ├── ClickHouseDbProviderFactoryTests.cs │ ├── ClickHouseParameterCollectionTests.cs │ ├── ClickHouseParameterTests.cs │ ├── ClickHouseTestsBase.cs │ ├── ClickHouseTypeInfoTests.cs │ ├── CommonUtilsTests.cs │ ├── EncodingFixture.cs │ ├── IndexedCollectionTests.cs │ ├── ListExtensionsTests.cs │ ├── Octonica.ClickHouseClient.Tests.csproj │ ├── ReadWriteBufferTests.cs │ ├── TestBox.cs │ ├── TestEnum.cs │ ├── TestIndexedCollection.cs │ ├── TestListWrappers.cs │ ├── TypeTests.cs │ └── xunit.runner.json └── Octonica.ClickHouseClient.sln ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUNIT *.VisualState.xml TestResult.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # JustCode is a .NET coding add-in .JustCode # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- Backup*.rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ ## ## Visual studio for Mac ## # globs Makefile.in *.userprefs *.usertasks config.make config.status aclocal.m4 install-sh autom4te.cache/ *.tar.gz tarballs/ test-results/ # Mac bundle stuff *.dmg *.app # content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore # General .DS_Store .AppleDouble .LSOverride # Icon must end with two \r Icon # Thumbnails ._* # Files that might appear in the root of a volume .DocumentRevisions-V100 .fseventsd .Spotlight-V100 .TemporaryItems .Trashes .VolumeIcon.icns .com.apple.timemachine.donotpresent # Directories potentially created on remote AFP share .AppleDB .AppleDesktop Network Trash Folder Temporary Items .apdisk # content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore # Windows thumbnail cache files Thumbs.db ehthumbs.db ehthumbs_vista.db # Dump file *.stackdump # Folder config file [Dd]esktop.ini # Recycle Bin used on file shares $RECYCLE.BIN/ # Windows Installer files *.cab *.msi *.msix *.msm *.msp # Windows shortcuts *.lnk # JetBrains Rider .idea/ *.sln.iml ## ## Visual Studio Code ## .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json # Test configuration /src/Octonica.ClickHouseClient.Tests/clickHouse.dbconfig # Benchmark configuration /src/Octonica.ClickHouseClient.Benchmarks/clickHouse.dbconfig ================================================ FILE: CHANGELOG.md ================================================ ### Octonica.ClickHouseClient Next Version, Unscheduled #### New Feature * Add method `ClickHouseDataReader.ConfigureColumnReader` which sets a value cast callback function for the column. The callback function is invoked whenever the reader reads a non-null value from the column. This feature could be useful in cases when ClcikHouseClient doesn't have a built-in type conversion rule, for expample, `double -> decimal`, `Guid -> string` or `uint -> int`. #### Improvement * Add method `ClickHouseParameterCollection.AddRange` receiveng any enumerable collection of `ClickHouseParameter`. ### Octonica.ClickHouseClient v2.2.11, 2023-01-11 #### Bug Fix * Fix null reference exception when the garbage collector calls the finalizer for `ClickHouseConnection` ([#70](https://github.com/Octonica/ClickHouseClient/issues/70)). ### Octonica.ClickHouseClient v2.2.10, 2022-12-30 #### New Feature * Add support for the type `Bool` ([#56](https://github.com/Octonica/ClickHouseClient/issues/56)). #### Bug Fix * Return the correct non-generic enumerator for `ClickHouseParameterCollection` (PR [#65](https://github.com/Octonica/ClickHouseClient/pull/65)). #### Improvement * Remove arguments with default values from constructors of `ClickHouseConnection`. It makes possible to call the constructor `ClickHouseConnection(sting)` using reflection ([#54](https://github.com/Octonica/ClickHouseClient/issues/54)). ### Octonica.ClickHouseClient v2.2.9, 2022-04-27 #### New Feature * New mode of passing parameters to a query - `Interpolate`. In this mode values are interpolated into the query text as constant literals. Parameter mode can be set for a connection (the property `ParametersMode` in the connection string), for a command (the property `ClickHouseCommand.ParametersMode`) or for a single parameter (`ClickHouseParameter.ParameterMode`) ([#49](https://github.com/Octonica/ClickHouseClient/issues/49), PR [#42](https://github.com/Octonica/ClickHouseClient/pull/42)). #### Improvement * Set `DateTimeKind.Unspecified` when cast a value of ClickHouse types `Date` and `Date32` to the .NET type `DateTime` (PR [#45](https://github.com/Octonica/ClickHouseClient/pull/45)). ### Octonica.ClickHouseClient v2.2.8, 2022-01-09 #### Bug Fix * Fix getting a time zone from IANA code. This fix is only applicable to the .NET 6 version of ClickHouseClient running on Windows ([#40](https://github.com/Octonica/ClickHouseClient/issues/40)). #### Improvement * Make possible to open a connection to the server with an unrecognized time zone. The `TimeZoneNotFoundException` may be thrown later when reading the column of type `DateTime` or `DateTime64` ([#40](https://github.com/Octonica/ClickHouseClient/issues/40)). #### Miscellaneous * Default protocol revision is set to 54452. This change was made because the minimal protocol revison with profile events was updated in the ClickHouse v21.12. ### Octonica.ClickHouseClient v2.2.7, 2021-12-04 #### New Feature * .NET 6.0 support ([#33](https://github.com/Octonica/ClickHouseClient/issues/33)): * New API for time zones. Remove dependency from the package TimeZoneConverter; * Change mapping of the ClickHouse type `Date` from `DateTime` to `DateOnly`. This affects the behavior of methods `ClickHouseDataReader.GetValue` and `ClickHouseDataReader.GetValues`; * Add the method `ClickHouseDataReader.GetDate` for reading values of types `Date` and `Date32`. * Add methods to the `ClickHouseDataReader` for reading values of well-known types ([#38](https://github.com/Octonica/ClickHouseClient/issues/38)): * `GetBigInteger`; * `GetIPAddress`; * `GetSByte`; * `GetUInt16`; * `GetUInt32`; * `GetUInt64`. * Add support for the type `Date32` ([#36](https://github.com/Octonica/ClickHouseClient/issues/36)). * Add support for profile events. Profile events are disabled by default. To enable it set the value of the property `ClickHouseCommand.IgnoreProfileEvents` to `false`. Please note that the method `ClickHouseDataReader.NextResult` (or `NextResultAsync`) should be called for switching between regular data and profile events. #### Bug Fix * Fix reading empty values of the type `LowCardinality(String)` ([#37](https://github.com/Octonica/ClickHouseClient/issues/37)). #### Miscellaneous * Default protocol revision is set to 54450. ### Octonica.ClickHouseClient release v2.1.2, 2021-11-07 #### New Feature * Add support for Transport Layer Security (TLS) connection ([#35](https://github.com/Octonica/ClickHouseClient/issues/35)). ### Octonica.ClickHouseClient release v2.1.1, 2021-09-16 #### Backward Incompatible Change * Classes from the namespace `Octonica.ClickHouseClient` that are now sealed and therefore can't be inherited: * `ClickHouseColumnWriter`; * `ClickHouseCommand`; * `ClickHouseConnection`; * `ClickHouseConnectionSettings`; * `ClickHouseDataReader`; * `ClickHouseParameter`; * `ClickHouseParameterCollection`; * `ClickHouseServerInfo`; * `ClickHouseTableColumnCollection`; * `ClickHouseTableProvider`; * `ClickHouseTableProviderCollection`. * Classes, enums and interfaces from the namespace `Octonica.ClickHouseClient.Protocol` that are no longer public: * `BlockFieldCodes`; * `CompressionAlgorithm`; * `IClickHouseTableWriter`; * `NullableObjTableColumn`. * The class `ClickHouseColumnSettings` was moved from the namespace `Octonica.ClickHouseClient.Protocol` to the namespace `Octonica.ClickHouseClient`. * The class `Revisions` from the namespace `Octonica.ClickHouseClient.Protocol` was renamed to `ClickHouseProtocolRevisions`. #### Improvement * Add XML documentation comments to the NuGet package. #### Bug Fix * Fix reading and writing values of the type `Array(LowCardinality(T))` ([#34](https://github.com/Octonica/ClickHouseClient/issues/34)). * Fix error handling for `ClickHouseColumnWriter`. ### Octonica.ClickHouseClient release v1.3.1, 2021-07-13 #### New Feature * Values of the type `FixedString` can be converted to the type `char[]`. * Values of the type `String` can be converted to types `char[]` and `byte[]`. #### Miscellaneous * Basic interfaces of the column reader were modified. Despite these interfaces are public, they are supposedly used only by an internal part of ClickHouseClient. ### Octonica.ClickHouseClient release v1.2.1, 2021-06-25 ### New Feature * Add support for user-defined tables in queries. New property `ClickHouseCommand.TableProviders` provides access to a collection of user-defined tables associated with the command. See the section *'Table-valued parameters'* of [Parameters](docs/Parameters.md) for details ([#24](https://github.com/Octonica/ClickHouseClient/issues/24)). * Add property `ClickHouseColumnWriter.MaxBlockSize`. This property allows to set the maximal number of rows which can be sent to the server as a single block of data. If an input table contains more rows than `MaxBlockSize` it will be sent to the server by parts ([#26](https://github.com/Octonica/ClickHouseClient/issues/26)). * Add support for the experimental type `Map(key, value)` ([#31](https://github.com/Octonica/ClickHouseClient/issues/31)). * Add support for long integer types `Int128`, `UInt128`, `Int256` and `UInt256` ([#27](https://github.com/Octonica/ClickHouseClient/issues/27)). #### Improvement * Improve performance of reading and writing values of primitive types. * Improve connection state management and error handling. ### Octonica.ClickHouseClient release v1.1.13, 2021-05-29 #### Bug Fix * Fix conversion from `System.Guid` to `UUID`. This bug affected `ClickHouseColumnWriter`. It caused writing of corrupted values to a column of type `UUID` ([#29](https://github.com/Octonica/ClickHouseClient/issues/29)). #### New Feature * Add method `ClickHouseConnection.TryPing`. This method allows to send 'Ping' message and wait for response from the server. #### Improvement * Add cast from `UInt8` to `bool`. `ClickHouseDataReader.GetBoolean` no longer throws an exception for values of type `UInt8`. ### Octonica.ClickHouseClient release v1.1.12, 2021-05-19 #### Backward Incompatible Change * `ClickHouseDataReader.GetField` and `ClickHouseColumnWriter.GetField` now return `typeof(T)` instead of `typeof(Nullable)` for nullable fields. It is possible to get original type of a column from field's type info: `ClickClickHouseDataReader.GetFieldTypeInfo(int ordinal).GetFieldType()`. * Stricter column type check. `ClickHouseColumnWriter` throws an exception when a type of a column is ambiguous (for example, a column's type implements both `IReadOnlyList` and `IReadOnlyList`). #### New Feature * Add support for named tuples. * Add a way to explicitly set a type of a column. The type could be defined in `ClickHouseColumnSettings`. `ClickHouseDataReader` will try to convert a column's value to this type. `ClickHouseColumnWriter` will expect a column to be a collection of items of this type. * Add support for `IReadOnlyList`, `IList`, `IEnumerable` and `IAsyncEnumerable` to `ClickHouseColumnWriter` ([#21](https://github.com/Octonica/ClickHouseClient/issues/21)). #### Bug Fix * Add recognition of escape sequences in enum's item names. ### Octonica.ClickHouseClient release v1.1.9, 2021-05-07 #### New Feature * Parameters in the format `@paramName` are supported in the text of a query ([#19](https://github.com/Octonica/ClickHouseClient/issues/19)). ### Octonica.ClickHouseClient release v1.1.8, 2021-04-25 #### New Feature * `ClickHouseCommand.ExecuteDbDataReader` supports non-default command behavior ([#18](https://github.com/Octonica/ClickHouseClient/issues/18)). * Added method `GetTypeArgument` to the interface `IClickHouseTypeInfo`. This method allows to get additional arguments of the type (scale, precision, timezone, size). ### Octonica.ClickHouseClient release v1.1.7, 2021-03-15 #### Bug Fix * Fixed error handling for `ClickHouseConnection.Open`. The socket was not properly disposed when error occurred during opening a connection. ### Octonica.ClickHouseClient release v1.1.6, 2021-03-08 #### Backward Incompatible Change * ClickHouseParameter can't be added to several parameter collections. Use the method `ClickHouseParameter.Clone` to create a parameter's copy which doesn't belong to the collection. #### New Feature * Octonica.ClickHouseClient for .NET 5.0 was added to NuGet package. * Added ClickHouseDbProviderFactory which implements DbProviderFactory. * `ReadOnlyMemory` or `Memory` can be used instead of `string` when writing values to ClickHouse. * `ReadOnlyMemory` or `Memory` can be used instead of `T[]` (array of `T`) when writing values to ClickHouse. #### Bug Fix * Fixed possible race condition when disposing a connection from different threads ([#16](https://github.com/Octonica/ClickHouseClient/issues/16)). #### Improvement * Improved implementation of various classes from `System.Data.Common` namespace, such as `DbConnection`, `DbCommand` and `DbParameter`. ### Octonica.ClickHouseClient release v1.0.17, 2020-12-10 #### Bug Fix * Fixed execution of queries which affect large (greater than 2^31) number of rows ([#15](https://github.com/Octonica/ClickHouseClient/issues/15)). * Fixed comparison of parameter's names in ClickHouseParameterCollection. #### Improvement * Added public method `ClickHouseParameter.IsValidParameterName` which allows to check if the string can be used as the name of a parameter. ### Octonica.ClickHouseClient release v1.0.14, 2020-12-02 #### Bug Fix * The driver was incompatible with ClickHouse v2.10 and higher. * Fixed writing columns from a source which contains more rows than `rowCount`. * Fixed writing columns from a source which implements `IList` but doesn't implement `IReadOnlyList`. ### Octonica.ClickHouseClient release v1.0.13, 2020-11-11 #### Backward Incompatible Change * The default name of the client changed from `Octonica.ClickHouse` to `Octonica.ClickHouseClient`. #### New Feature * Added type `DateTime64`. * Implemented methods `NextResult` and `NextResultAsync` in `ClickHouseDataReader`. These methods can be used to read totals and extremes ([#11](https://github.com/Octonica/ClickHouseClient/issues/11)). * Added `Extremes` property to `ClickHouseCommand`. It allows to toggle `extremes` setting for the query. * Added `TimeZone` property to `ClickHouseParameter`. It allows to specify the timezone for datetime types. * Array can be used as the value of command parameter. Added properties `IsArray` and `ArrayRank` to `ClickHouseParameter` ([#14](https://github.com/Octonica/ClickHouseClient/issues/14)). #### Bug Fix * The type `UInt64` was mapped to the type `UInt32` in the command parameter. #### Improvement * Detection of attempts to connect to ClickHouse server with HTTP protocol ([#10](https://github.com/Octonica/ClickHouseClient/issues/10)). * `ReadWriteTimeout` is respected in async network operations if `cancellationToken` is not defined (i.e. `CanellationToken.None`). #### Miscellaneous * Default protocol revision is set to 54441. ================================================ FILE: LICENSE ================================================ Copyright 2019-2026 Octonica Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2026 Octonica Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ ClickHouse .NET Core driver =============== This is an implementation of .NET Core driver for ClickHouse in a form of ADO.NET DbProvider API. This driver supports all ADO.NET features (with some exclusions like transaction support). ### Features * supports binary protocol * compression (send and recieve) * timezones * most clickhouse [column types](docs/TypeMapping.md) are supported ([aggregating ones](https://clickhouse.tech/docs/en/sql_reference/data_types/aggregatefunction/) are under development) * full support for .net async ADO.NET API * no unsafe code * ~~tested~~- used in production * c# named tuple and record support * [Dapper](https://dapperlib.github.io/Dapper/) support (example in [#19](https://github.com/Octonica/ClickHouseClient/issues/19)) * [Linq To DB](https://github.com/linq2db/linq2db) support ### Usage Install from [NuGet](https://www.nuget.org/packages/Octonica.ClickHouseClient/): ``` dotnet add package Octonica.ClickHouseClient ``` ConnectionString syntax: `Host=;Port=;Database=;Password=`, e.g. `"Host=127.0.0.1;Password=P@ssw0rd; Database=db` additionally, if you want to build a connection string via code you can use `ClickHouseConnectionStringBuilder`. Entry point for API is ADO .NET DbConnection Class: `Octonica.ClickHouse.ClickHouseConnection`. ### Extended API In order to provide non-ADO.NET complaint data manipulation functionality, proprietary [ClickHouseColumnWriter](docs/ClickHouseColumnWriter.md) API exists. Entry point for API is `ClickHouseConnection.CreateColumnWriter()` method. #### Simple SELECT async verison ```csharp var sb = new ClickHouseConnectionStringBuilder(); sb.Host = "127.0.0.1"; using var conn = new ClickHouseConnection(sb); await conn.OpenAsync(); var currentUser = await conn.CreateCommand("select currentUser()").ExecuteScalarAsync(); ``` #### Insert data with parameters ```csharp var sb = new ClickHouseConnectionStringBuilder(); sb.Host = "127.0.0.1"; using var conn = new ClickHouseConnection(sb); conn.Open(); using var cmd = conn.CreateCommand("INSERT INTO table_you_just_created SELECT {id}, {dt}"); cmd.Parameters.AddWithValue("id", Guid.NewGuid()); cmd.Parameters.AddWithValue("dt", DateTime.Now, System.Data.DbType.DateTime); var _ = cmd.ExecuteNonQuery(); ``` For more information see [Parameters](docs/Parameters.md). #### Bulk insert ```csharp using var conn = new ClickHouseConnection("Host=127.0.0.1"); conn.Open(); using var cmd = conn.CreateCommand("CREATE TABLE IF NOT EXISTS table_with_two_fields(id Int32, name String) engine Memory"); await cmd.ExecuteNonQueryAsync(); //generate values List ids = Enumerable.Range(1, 10_000).ToList(); List names = ids.Select(i => $"Name #{i}").ToList(); //insert data await using (var writer = await conn.CreateColumnWriterAsync("insert into table_with_two_fields(id, name) values", default)) { await writer.WriteTableAsync(new object[] { ids, names }, ids.Count, default); } ``` ### Build requirements In order to build the driver you need to have .NET SDK 5.0 or higher. ================================================ FILE: docs/ClickHouseColumnWriter.md ================================================ # ClickHouseColumnWriter `ClickHouseColumnWriter` is a class dedicated for writing arbitrary large amount of rows to a table. It writes tables in a columnar layout. It means that the table consists of several columns and each column contains a list of values (cells) of a particular type. All columns of the table must contain the same number of cells. To create a writer call the method `ClickHouseConnection.CreateColumnWriter` (or `ClickHouseConnection.CreateColumnWriterAsync`). ```C# using var connection = ClickHouseConnection(connectionStr); connection.Open(); using var writer = connection.CreateColumnWriter("INSERT INTO some_table VALUES"); ``` Please, note that the `INSERT` query for the writer must end with `VALUES` keyword, but without actual list of values. Some methods of `ClickHouseColumnWriter` are similar to methods of `ClickHouseDataReader`: `GetName`, `GetOrdinal`, `GetFieldType`, `ConfigureColumn` and other methods for manipulating column metadata. Here is the method for writing tables: ```C# void WriteTable(IReadOnlyList columns, int rowCount) ``` A column could be of any type which implements one of interfaces: * `IReadOnlyList`; * `IList`; * `IEnumerable`; * `IAsyncEnumerable` (supported only by `WriteTableAsync`); * `IEnumerable`. The number of columns must be equal to the number of columns in the initial `INSERT` query. Columns may have different number of rows, but not less than `rowCount`. Columns must be passed in the order defined by the query. There is an overload of `WriteTable` which distinguishes columns by their names: ```C# void WriteTable(IReadOnlyDictionary columns, int rowCount) ``` ## Examples Assume there is a table `some_table`. ```C# using var connection = new ClickHouseConnection(connectionStr); connection.Open(); var cmd = connection.CreateCommand("CREATE TABLE some_table(id Int32, str Nullable(String), dt DateTime, val Decimal64(4)) ENGINE = Memory"); cmd.ExecuteNonQuery(); ``` ### Write ordered columns ```C# var id = new List(); var str = new List(); var dt = new List(); var val = new List(); /* * Fill lists id, str, dt and val with actual values */ await using var connection = new ClickHouseConnection(connectionStr); await connection.OpenAsync(); await using var writer = connection.CreateColumnWriter("INSERT INTO some_table VALUES"); var columns = new object[writer.FieldCount]; columns[writer.GetOrdinal("id")] = id; columns[writer.GetOrdinal("str")] = str; columns[writer.GetOrdinal("dt")] = dt; columns[writer.GetOrdinal("val")] = val; var rowCount = id.Count; await writer.WriteTableAsync(columns, rowCount, CancellationToken.None); ``` ### Write named columns ```C# var id = new List(); var dt = new List(); var val = new List(); /* * Fill lists id, dt and val with actual values */ await using var connection = new ClickHouseConnection(connectionStr); await connection.OpenAsync(); await using var writer = connection.CreateColumnWriter("INSERT INTO some_table(id, dt, val) VALUES"); var columns = new Dictionary { ["id"] = id, ["dt"] = dt, ["val"] = val }; var rowCount = id.Count; await writer.WriteTableAsync(columns, rowCount, CancellationToken.None); ``` ================================================ FILE: docs/Parameters.md ================================================ # Parameters ClickHouseClient's implementation of parameters is compliant with ADO.NET. API for working with parameters intended to be familiar to users of other ADO.NET drivers. Each command (`Octonica.ClickHouseClient.ClickHouseCommand`) contains a collection of parameters. This collection can be acquired with the property `Parameters`. Parameters from this collection can be referenced in a query. ## Parameter format As specified in [Queries with Parameters](https://clickhouse.tech/docs/en/interfaces/cli/#cli-queries-with-parameters) the default format of the parameter is `{:}`. However, the type can be derived from the parameter's settings, which allows to omit `:` part and declare parameter just as `{}`. ClickHouseClient also supports parameters in MSSQL-like format: `@`. Here is an example demonstrating different namestyles of parameters: ```C# using var connection = new ClickHouseConnection(connectionStr); connection.Open(); var cmd = connection.CreateCommand("SELECT number/{value:Decimal64(2)} FROM numbers(100000) WHERE number >= @min AND number <= {max}"); cmd.Parameters.AddWithValue("value", 100); cmd.Parameters.AddWithValue("{min}", 1000); cmd.Parameters.AddWithValue("@max", 2000); using var reader = cmd.ExecuteReader(); while(reader.Read()) { // Reading the data } ``` ## Parameter settings The settings of the parameter usually can be detected based on the type of the parameter's value. It is possible to override auto-detected settings in cases when auto-detection fails or when the required type of the parameter doesn't match to the type of the parameter's value. When settings are overridden ClickHouseClient will try to convert parameter's value to the requested type. There are settings inherited from `System.Data.Common.DbParameter`: 1. `DbType`. The type of the parameter. `System.Data.DbType` is a subset of `Octonica.ClickHouseClient.ClickHouseDbType`. For any value of the property `ClickHouseDbType` which can't be mapped to the type `System.Data.DbType` this property returns `DbType.Object`; 2. `IsNullable`. Indicates whether the parameter accept `NULL`; 3. `Precision`. Defines the precision for `Decimal` or `DateTime64`; 4. `Scale`. Defines the scale for `Decimal`; 5. `Size`. Defines the size for `FixedString`. And there are additional settings supported by `Octonica.ClickHouseClient.ClickHouseParameter`: 1. `ArrayRank`. The number of dimensions in the array. Zero for non-arrays; 2. `ClickHouseDbType`. The type of the parameter; 3. `IsArray`. Indicates whether the parameter is an array, i.e. `ArrayRank > 0`; 4. `StringEncoding`. Defines the encoding which will be used for strings; 5. `TimeZone`. Defines the timezone for `DateTime` or `DateTime64`. ## Table-valued parameters To be fair, a parameter can't be a table. However, ClickHouse allows to pass arbitrary tables with a query. These tables can be referenced in the query without special syntax. The tables and parameters are stored separately in `Octonica.ClickHouseClient.ClickHouseCommand`. The collection of tables can be acquired with the property `TableProviders` of the command. The basic interface for client-defined table is `Octonica.ClickHouseClient.IClickHouseTableProvider`. A class implementing this interface should provide a table in a columnar format. There is a default implementation of this interface: `Octonica.ClickHouseClient.ClickHouseTableProvider`. This class allows to pass a table in a way similar to [ClickHouseColumnWriter](ClickHouseColumnWriter.md). Here is a simple example demonstrating how to pass a client-defined table to a query: ```C# using var connection = new ClickHouseConnection(connectionStr); connection.Open(); var cmd = connection.CreateCommand("SELECT ptable.id, ptable.user, ptable.ip FROM ptable"); var users = new[] {"user1", "user2", "admin1", "admin2"}; var ips = new[] {"1.1.1.1", "2.2.2.2", "127.0.0.1", "::ffff:192.0.2.1"}; var pTableProvider = new ClickHouseTableProvider("ptable", users.Length); pTableProvider.AddColumn("id", Enumerable.Range(1, users.Length)); pTableProvider.AddColumn("user", users); // The settings of the column are similar to the settings of parameter var ipColumn = pTableProvider.AddColumn("ip", ips); ipColumn.ClickHouseDbType = ClickHouseDbType.IpV6; ipColumn.IsNullable = false; cmd.TableProviders.Add(pTableProvider); using var reader = cmd.ExecuteReader(); while(reader.Read()) { // Reading the data } ``` And here is a bit more practical example demonstrating how a temporary table can be used with `IN` clause: ```C# using var connection = new ClickHouseConnection(connectionStr); connection.Open(); var cmd = cn.CreateCommand("SELECT toInt32(number) FROM numbers(100000) WHERE number IN param_table"); var tableProvider = new ClickHouseTableProvider("param_table", 100); tableProvider.AddColumn(Enumerable.Range(500, int.MaxValue / 2)); cmd.TableProviders.Add(tableProvider); using var reader = cmd.ExecuteReader(); while(reader.Read()) { // Reading only 100 rows } ``` ## Implementation details Unfortunately, parameters are not supported by the ClickHouse binary protocol. Which means that it's a client-side feature. ClickHouseClient passes parameters to the server as a table with one row. The name of this table is unique for each query. It is generated based on `Guid` so there should be no collision with names of existing tables. ClickHouseClient analyzes the query and substitutes parameters with `SELECT` subquery. For example, the query ```SQL SELECT * FROM some_table WHERE id = {id:UInt32} ``` will be transformed before sending to the server to ```SQL SELECT * FROM some_table WHERE id = (CAST((SELECT _b3dcef95634b4fcfbf67624a39ce2e85.id FROM _b3dcef95634b4fcfbf67624a39ce2e85) AS UInt32)) ``` where `_b3dcef95634b4fcfbf67624a39ce2e85` is the name of the table with parameters. ================================================ FILE: docs/TypeMapping.md ================================================ # Type mappings **ClickHouse type**. The type of the column. **Default type**. This is the type returned by `Octonica.ClickHouseClient.ClickHouseDataReader.GetFieldType(int ordinal)`. The method `Octonica.ClickHouseClient.ClickHouseDataReader.GetValue(int ordinal)` returns either a value of the default type or `System.DBNull`. **Supported types**. The value can be converted to one of this types. **ClickHouseDataReader's method**. The method dedicated to the default type. You can get the value of one of supported types by calling `GetFieldValue(int ordinal)` or `GetFieldValue(int ordinal, T? nullValue)`. The latter doesn't throw an error on NULL value. | ClickHouse type | Default type | Supported types | ClickHouseDataReader's method | |---|---|---|---| | Int8 | sbyte | short, int, long | `GetSByte` | | Int16 | short | int, long | `GetInt16` | | Int32 | int | long | `GetInt32` | | Int64 | long | | `GetInt64` | | Int128 | System.Numerics.BigInteger | | `GetBigInteger` | | Int256 | System.Numerics.BigInteger | | `GetBigInteger` | | UInt8 | byte | ushort, uint, ulong, int, long | `GetByte` | | UInt16 | ushort | uint, ulong, int, long | `GetUInt16` | | UInt32 | uint | ulong, long | `GetUInt132` | | UInt64 | ulong | | `GetUInt64` | | UInt128 | System.Numerics.BigInteger | | `GetBigInteger` | | UInt256 | System.Numerics.BigInteger | | `GetBigInteger` | | Float32 | float | double | `GetFloat` | | Float64 | double | | `GetDouble` | | Decimal | decimal | | `GetDecimal` | | Date\* | System.DateOnly | System.DateTime | `GetDate` | | Date32\* | System.DateOnly | System.DateTime | `GetDate` | | DateTime | System.DateTimeOffset | System.DateTime | `GetDateTimeOffset` | | DateTime64 | System.DateTimeOffset | System.DateTime | `GetDateTimeOffset` | | String | string | char[], byte[] | `GetString` | | FixedString | byte[] | string, char[] | | | UUID | System.Guid | | `GetGuid` | | IPv4 | System.Net.IPAddress | string, int, uint | `GetIPAddress` | | IPv6 | System.Net.IPAddress | string | `GetIPAddress` | | Enum8 | string | sbyte, short, int, long | | | Enum16 | string | short, int, long | | | Nothing | System.DBNull | | `GetValue` | | Nullable(T) | T? | | | | Array(T) | T[] | | | | Tuple(T1, ... Tn) | System.Tuple | System.ValueTuple | | | LowCardinality | T | | The method for `T` | | Map(TKey, TValue) | System.Collections.Generic.KeyValuePair[] | System.Tuple[], System.ValueTuple[] | | \* The type `System.DateOnly` is available since .NET 6.0. For previous .NET versions dates are mapped to `System.DateTime`. ================================================ FILE: src/ConnectionSettingsHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.IO; namespace Octonica.ClickHouseClient { internal static class ConnectionSettingsHelper { public static ClickHouseConnectionSettings GetConnectionSettings(Action? updateSettings = null) { return GetConnectionSettingsInternal(updateSettings).settings; } public static string GetConnectionString() { return GetConnectionSettingsInternal(null).connectionString; } public static string GetConnectionString(Action updateSettings) { var settings = GetConnectionSettings(updateSettings); var builder = new ClickHouseConnectionStringBuilder(settings); return builder.ConnectionString; } private static (ClickHouseConnectionSettings settings, string connectionString) GetConnectionSettingsInternal(Action? updateSettings) { const string envVariableName = "CLICKHOUSE_TEST_CONNECTION"; const string configFileName = "clickHouse.dbconfig"; const string conStrExample = "host=clickhouse.example.com; port=9000; user=default;"; var configTextFromEnvVar = Environment.GetEnvironmentVariable(envVariableName); if (configTextFromEnvVar != null) { try { var builder = new ClickHouseConnectionStringBuilder(configTextFromEnvVar); updateSettings?.Invoke(builder); return (builder.BuildSettings(), configTextFromEnvVar); } catch (Exception ex) { throw new InvalidOperationException($"The connection string from the environment variable '{envVariableName}' is not valid. Connection string example: '{conStrExample}'. {ex.Message}", ex); } } string configPath = Path.Combine(AppContext.BaseDirectory, configFileName); if (!File.Exists(configPath)) { throw new InvalidOperationException( "The connection string is required. " + $"Please, set the environment variable '{envVariableName}' or write the connection string to the file '{configFileName}'. " + $"Connection string example: '{conStrExample}'."); } string configText = File.ReadAllText(configPath); try { var builder = new ClickHouseConnectionStringBuilder(configText); updateSettings?.Invoke(builder); return (builder.BuildSettings(), configText); } catch (Exception ex) { throw new InvalidOperationException($"The connection string from the file '{configFileName}' is not valid. Connection string example: '{conStrExample}'. {ex.Message}", ex); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseBinaryProtocolReader.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { internal class ClickHouseBinaryProtocolReader: IDisposable { private readonly ReadWriteBuffer _buffer; private readonly Stream _stream; private readonly int _bufferSize; private CompressionAlgorithm _currentCompression; private CompressionDecoderBase? _compressionDecoder; public ClickHouseBinaryProtocolReader(Stream stream, int bufferSize) { _buffer = new ReadWriteBuffer(bufferSize); _stream = stream ?? throw new ArgumentNullException(nameof(stream)); _bufferSize = bufferSize; } internal void BeginDecompress(CompressionAlgorithm algorithm) { if (algorithm != CompressionAlgorithm.None) { if (_compressionDecoder != null && _compressionDecoder.Algorithm != algorithm) { _compressionDecoder?.Dispose(); _compressionDecoder = null; } else { _compressionDecoder?.Reset(); } } switch (algorithm) { case CompressionAlgorithm.None: _currentCompression = algorithm; return; case CompressionAlgorithm.Lz4: if (_compressionDecoder == null) _compressionDecoder = new Lz4CompressionDecoder(_bufferSize); _currentCompression = algorithm; break; default: throw new ArgumentOutOfRangeException(nameof(algorithm), algorithm, null); } } internal void EndDecompress() { _currentCompression = CompressionAlgorithm.None; } public async ValueTask ReadString(bool async, CancellationToken cancellationToken) { var size = await ReadSize(async, cancellationToken); if (size == 0) return string.Empty; ReadOnlySequence readResult; do { readResult = await Read(async, cancellationToken); if (readResult.Length >= size) break; AdvanceReader(readResult, 0); await Advance(async, cancellationToken); } while (true); string result; var encoding = Encoding.UTF8; var stringSpan = readResult.Slice(readResult.Start, readResult.GetPosition(size)); if (stringSpan.IsSingleSegment) { result = encoding.GetString(stringSpan.FirstSpan); } else { var buffer = stringSpan.ToArray(); result = encoding.GetString(buffer); } AdvanceReader(readResult, (int) stringSpan.Length); return result; } public async ValueTask Read7BitInt32(bool async, CancellationToken cancellationToken) { var longValue = await Read7BitInteger(async, cancellationToken); if (longValue > uint.MaxValue) throw new FormatException(); //TODO: exception return unchecked((int) longValue); } public ValueTask Read7BitUInt64(bool async, CancellationToken cancellationToken) { return Read7BitInteger(async, cancellationToken); } public async ValueTask ReadInt32(bool async, CancellationToken cancellationToken) { do { var readResult = await Read(async, cancellationToken); if (readResult.Length < sizeof(int)) { AdvanceReader(readResult, 0); await Advance(async, cancellationToken); continue; } int result; if (readResult.FirstSpan.Length >= sizeof(int)) result = BitConverter.ToInt32(readResult.FirstSpan); else { var tmpArr = readResult.Slice(0, sizeof(int)).ToArray(); result = BitConverter.ToInt32(tmpArr, 0); } AdvanceReader(readResult, sizeof(int)); return result; } while (true); } public async ValueTask ReadSize(bool async, CancellationToken cancellationToken) { var longValue = await Read7BitInteger(async, cancellationToken); if (longValue > int.MaxValue) throw new FormatException(); //TODO: exception return (int) longValue; } public async ValueTask ReadBool(bool async, CancellationToken cancellationToken) { return await ReadByte(async, cancellationToken) != 0; } public async ValueTask ReadByte(bool async, CancellationToken cancellationToken) { var readResult = await Read(async, cancellationToken); var result = readResult.FirstSpan[0]; AdvanceReader(readResult, 1); return result; } private async ValueTask Read7BitInteger(bool async, CancellationToken cancellationToken) { do { var readResult = await Read(async, cancellationToken); if (!TryRead7BitInteger(readResult, out var result, out var bytesRead)) { AdvanceReader(readResult, 0); await Advance(async, cancellationToken); } else { AdvanceReader(readResult, bytesRead); return result; } } while (true); } public async ValueTask ReadRaw(Func, SequenceSize> readBytes, bool async, CancellationToken cancellationToken) { if (readBytes == null) throw new ArgumentNullException(nameof(readBytes)); var readResult = await Read(async, cancellationToken); var size = readBytes(readResult); AdvanceReader(readResult, size.Bytes); return size; } public async ValueTask SkipBytes(int bytesCount, bool async, CancellationToken cancellationToken) { if (bytesCount < 0) throw new ArgumentException("The number of bytes for is negative.", nameof(bytesCount)); if (bytesCount == 0) return; var c = bytesCount; while (c > 0) { var readResult = await Read(async, cancellationToken); var consumed = Math.Min(c, (int)readResult.Length); AdvanceReader(readResult, consumed); c -= consumed; } } internal bool TryPeekByte(out byte value) { if (_currentCompression != CompressionAlgorithm.None) throw new NotImplementedException(); var readResult = _buffer.Read(); if (readResult.IsEmpty) { value = 0; return false; } value = readResult.FirstSpan[0]; return true; } public async ValueTask ReadMessage(int protocolRevision, bool throwOnUnknownMessage, bool async, CancellationToken cancellationToken) { var messageCode = (ServerMessageCode) await Read7BitInt32(async, cancellationToken); switch (messageCode) { case ServerMessageCode.Hello: return await ServerHelloMessage.Read(this, protocolRevision, async, cancellationToken); case ServerMessageCode.Data: case ServerMessageCode.Totals: case ServerMessageCode.Extremes: return await ServerDataMessage.Read(this, messageCode, async, cancellationToken); case ServerMessageCode.Error: return await ServerErrorMessage.Read(this, async, cancellationToken); case ServerMessageCode.Progress: return await ServerProgressMessage.Read(this, protocolRevision, async, cancellationToken); case ServerMessageCode.Pong: return ServerPongMessage.Instance; case ServerMessageCode.EndOfStream: return ServerEndOfStreamMessage.Instance; case ServerMessageCode.ProfileInfo: return await ServerProfileInfoMessage.Read(this, async, cancellationToken); case ServerMessageCode.TableColumns: return await ServerTableColumnsMessage.Read(this, async, cancellationToken); case ServerMessageCode.TableStatusResponse: case ServerMessageCode.Log: case ServerMessageCode.PartUuids: case ServerMessageCode.ReadTaskRequest: case ServerMessageCode.MergeTreeAllRangesAnnouncement: case ServerMessageCode.MergeTreeReadTaskRequest: throw new NotImplementedException($"A message of type \"{messageCode}\" is not supported."); case ServerMessageCode.ProfileEvents: return await ServerDataMessage.Read(this, messageCode, async, cancellationToken); case ServerMessageCode.TimezoneUpdate: return await ServerTimeZoneUpdateMessage.Read(this, async, cancellationToken); default: if (throwOnUnknownMessage) throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Not supported message code (0x{messageCode:X}) received from the server."); return new UnknownServerMessage(messageCode); } } private async ValueTask> Read(bool async, CancellationToken cancellationToken) { if (_currentCompression == CompressionAlgorithm.None) return await ReadFromPipe(async, cancellationToken); if (_compressionDecoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. An encoder is not initialized."); if (!_compressionDecoder.IsCompleted) await Advance(async, cancellationToken); var sequence = _compressionDecoder.Read(); while (sequence.IsEmpty) { await Advance(async, cancellationToken); sequence = _compressionDecoder.Read(); } return sequence; } private async ValueTask> ReadFromPipe(bool async, CancellationToken cancellationToken) { do { var readResult = _buffer.Read(); if (!readResult.IsEmpty) return readResult; await AdvanceBuffer(async, cancellationToken); } while (true); } private void AdvanceReader(ReadOnlySequence readResult, int consumedPosition) { if (_currentCompression == CompressionAlgorithm.None) { _buffer.ConfirmRead(consumedPosition); } else { if (_compressionDecoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. A decoder is not initialized."); _compressionDecoder.AdvanceReader(readResult.GetPosition(consumedPosition)); } } internal async ValueTask Advance(bool async, CancellationToken cancellationToken) { if (_currentCompression != CompressionAlgorithm.None) { if (_compressionDecoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. A decoder is not initialized."); if (_compressionDecoder.IsCompleted) { while (true) { var buffer = await ReadFromPipe(async, cancellationToken); var size = _compressionDecoder.ReadHeader(buffer); if (size >= 0) { _buffer.ConfirmRead(size); break; } await AdvanceBuffer(async, cancellationToken); } } while (!_compressionDecoder.IsCompleted) { var sequence = await ReadFromPipe(async, cancellationToken); var consumed = _compressionDecoder.ConsumeNext(sequence); _buffer.ConfirmRead(consumed); } return; } await AdvanceBuffer(async, cancellationToken); } private async ValueTask AdvanceBuffer(bool async, CancellationToken cancellationToken) { var buffer = _buffer.GetMemory(); int bytesRead; if (async) { if (cancellationToken == CancellationToken.None && _stream.ReadTimeout >= 0) { var timeout = TimeSpan.FromMilliseconds(_stream.ReadTimeout); using var tokenSource = new CancellationTokenSource(timeout); try { bytesRead = await _stream.ReadAsync(buffer, tokenSource.Token); } catch (OperationCanceledException ex) { throw new IOException($"Unable to read data from the transport connection: timeout exceeded ({timeout}).", ex); } } else { bytesRead = await _stream.ReadAsync(buffer, cancellationToken); } } else { bytesRead = _stream.Read(buffer.Span); cancellationToken.ThrowIfCancellationRequested(); } if (bytesRead == 0) throw new EndOfStreamException($"Reached an unexpected end of the server's response. {ClickHouseConnectionStringBuilder.DefaultClientName} expected at least one more byte in the response."); _buffer.ConfirmWrite(bytesRead); _buffer.Flush(); } public static bool TryRead7BitInteger(ReadOnlySequence sequence, out ulong value, out int bytesRead) { ulong result = 0; int i = 0, shiftSize = 0; foreach (var slice in sequence) { for (int j = 0; j < slice.Length; j++) { var byteValue = slice.Span[j]; result |= (byteValue & (ulong)0x7F) << shiftSize; i++; if ((byteValue & 0x80) == 0x80) { shiftSize += 7; if (shiftSize > sizeof(ulong) * 8 - 7) throw new FormatException(); //TODO: exception } else { value = result; bytesRead = i; return true; } } } value = 0; bytesRead = 0; return false; } public void Dispose() { _compressionDecoder?.Dispose(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseBinaryProtocolWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { internal class ClickHouseBinaryProtocolWriter : IDisposable { private readonly int _bufferSize; private readonly ReadWriteBuffer _buffer; private readonly Stream _stream; private CompressionAlgorithm _currentCompression; private CompressionEncoderBase? _compressionEncoder; public ClickHouseBinaryProtocolWriter(Stream stream, int bufferSize) { _buffer = new ReadWriteBuffer(bufferSize); _stream = stream ?? throw new ArgumentNullException(nameof(stream)); _bufferSize = bufferSize; } public async ValueTask Flush(bool async, CancellationToken cancellationToken) { if (_currentCompression != CompressionAlgorithm.None) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. The stream can't be flushed because it's compression is not completed."); _buffer.Flush(); var readResult = _buffer.Read(); if (readResult.IsEmpty) return; foreach (var buffer in readResult) { if (async) await WriteWithTimeoutAsync((networkStream, ct) => networkStream.WriteAsync(buffer, ct).AsTask(), cancellationToken); else _stream.Write(buffer.Span); } _buffer.ConfirmRead((int) readResult.Length); if (async) await WriteWithTimeoutAsync((networkStream, ct) => networkStream.FlushAsync(ct), cancellationToken); else _stream.Flush(); } public void Discard() { _buffer.Discard(); var readResult = _buffer.Read(); if (!readResult.IsEmpty) _buffer.ConfirmRead((int) readResult.Length); _currentCompression = CompressionAlgorithm.None; } public void BeginCompress(CompressionAlgorithm algorithm, int compressionBlockSize) { if (_compressionEncoder != null) { if (_compressionEncoder.Algorithm != algorithm) { _compressionEncoder.Dispose(); _compressionEncoder = null; } else { _currentCompression = _compressionEncoder.Algorithm; _compressionEncoder.Reset(); return; } } switch (algorithm) { case CompressionAlgorithm.None: break; case CompressionAlgorithm.Lz4: _currentCompression = algorithm; _compressionEncoder = new Lz4CompressionEncoder(_bufferSize, compressionBlockSize); break; default: throw new NotSupportedException($"Compression algorithm \"{algorithm}\" is not supported."); } } public void EndCompress() { _compressionEncoder?.Complete(_buffer); _currentCompression = CompressionAlgorithm.None; } public void WriteString(string value) { if (value == null) throw new ArgumentNullException(nameof(value)); var encoding = Encoding.UTF8; var length = encoding.GetByteCount(value); Write7BitInteger((uint) length); if (length == 0) return; var charSpan = value.AsSpan(); var byteSpan = GetSpan(length); var count = Encoding.UTF8.GetBytes(charSpan, byteSpan); Debug.Assert(count == length); Advance(length); } public SequenceSize WriteRaw(Func, SequenceSize> writeBytes) { return WriteRaw(0, writeBytes); } public SequenceSize WriteRaw(int sizeHint, Func, SequenceSize> writeBytes) { if (writeBytes == null) throw new ArgumentNullException(nameof(writeBytes)); SequenceSize size; var memory = sizeHint > 0 ? GetMemory(sizeHint) : GetMemory(); if (!memory.IsEmpty) { try { size = writeBytes(memory); } catch { Advance(0); throw; } if (size.Bytes > 0 || size.Elements > 0) { Advance(size.Bytes); return size; } } var bufferSize = _bufferSize; do { Advance(0); memory = GetMemory(bufferSize); try { size = writeBytes(memory); } catch { Advance(0); throw; } bufferSize *= 2; } while (size.Bytes == 0 && size.Elements == 0); Advance(size.Bytes); return size; } public void WriteInt32(int value) { var span = GetSpan(sizeof(int)); var success = BitConverter.TryWriteBytes(span, value); Debug.Assert(success); Advance(sizeof(int)); } public void WriteBool(bool value) { WriteByte(value ? (byte) 1 : (byte) 0); } public void WriteByte(byte value) { var buffer = GetSpan(1); buffer[0] = value; Advance(1); } public void WriteBytes(ReadOnlySpan bytes) { var span = GetSpan(bytes.Length); bytes.CopyTo(span); Advance(bytes.Length); } private async Task WriteWithTimeoutAsync(Func writeAsync, CancellationToken cancellationToken) { if (cancellationToken == CancellationToken.None && _stream.WriteTimeout >= 0) { var timeout = TimeSpan.FromMilliseconds(_stream.WriteTimeout); using var tokenSource = new CancellationTokenSource(timeout); try { await writeAsync(_stream, tokenSource.Token); } catch (OperationCanceledException ex) { throw new IOException($"Unable to write data to the transport connection: timeout exceeded ({timeout}).", ex); } } else { await writeAsync(_stream, cancellationToken); } } public void Write7BitInt32(int value) { var ulongValue = (ulong) unchecked((uint) value); Write7BitInteger(ulongValue); } private void Write7BitInteger(ulong value) { ulong v = value; int totalLength = 0; var buffer = GetSpan(10); for (int i = 0; i < buffer.Length; i++) { ++totalLength; if (v >= 0x80) { buffer[i] = (byte) (v | 0x80); v >>= 7; } else { buffer[i] = (byte) v; Advance(totalLength); return; } } Advance(totalLength); } private Span GetSpan(int sizeHint) { if (_currentCompression != CompressionAlgorithm.None) { if (_compressionEncoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. An encoder is not initialized."); return _compressionEncoder.GetSpan(sizeHint); } return _buffer.GetMemory(sizeHint).Span; } private Memory GetMemory(int sizeHint) { if (_currentCompression == CompressionAlgorithm.None) return _buffer.GetMemory(sizeHint); if (_compressionEncoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. An encoder is not initialized."); return _compressionEncoder.GetMemory(sizeHint); } private Memory GetMemory() { if (_currentCompression == CompressionAlgorithm.None) return _buffer.GetMemory(); if (_compressionEncoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. An encoder is not initialized."); return _compressionEncoder.GetMemory(); } private void Advance(int bytes) { if (_currentCompression != CompressionAlgorithm.None) { if (_compressionEncoder == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. An encoder is not initialized."); _compressionEncoder.Advance(bytes); } else { _buffer.ConfirmWrite(bytes); } } public static int TryWrite7BitInteger(Span buffer, ulong value) { ulong v = value; int count = 0; while (true) { if (buffer.Length == count) return 0; if (v >= 0x80) { buffer[count++] = (byte) (v | 0x80); v >>= 7; } else { buffer[count++] = (byte) v; break; } } return count; } public void Dispose() { _compressionEncoder?.Dispose(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseColumnSettings.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Text; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Represents additional column settings that affect the behavior of and . /// public class ClickHouseColumnSettings { private ITypeDispatcher? _columnTypeDispatcher; /// /// Gets encoding applied to strings when reading from the database or writing to the database. /// public Encoding? StringEncoding { get; } /// /// Gets the converter applied to enums. /// public IClickHouseEnumConverter? EnumConverter { get; } /// /// Gets the explicitly defined type of the column. This value overrides the type of the field for /// and . will try to convert a column's value to this type. /// will expect a collection of items of this type as input. /// public Type? ColumnType { get; } /// /// Initializes a new instance of class with the specified encoding. /// /// The encoding applied to strings when reading from the database or writing to the database. public ClickHouseColumnSettings(Encoding stringEncoding) { StringEncoding = stringEncoding ?? throw new ArgumentNullException(nameof(stringEncoding)); } /// /// Initializes a new instance of class with the specified enum converter. /// /// The converter applied to enums. public ClickHouseColumnSettings(IClickHouseEnumConverter enumConverter) { EnumConverter = enumConverter ?? throw new ArgumentNullException(nameof(enumConverter)); } /// /// Initializes a new instance of class with the specified column type. /// /// /// The explicitly defined type of the column. This value overrides the type of the field for /// and . will try to convert a column's value to this type. /// will expect a collection of items of this type as input. /// public ClickHouseColumnSettings(Type columnType) { ColumnType = columnType ?? throw new ArgumentException(nameof(columnType)); } /// /// Initializes a new instance of class with multiple specified parameters. /// /// The encoding applied to strings when reading from the database or writing to the database. /// The converter applied to enums. /// /// The explicitly defined type of the column. This value overrides the type of the field for /// and . will try to convert a column's value to this type. /// will expect a collection of items of this type as input. /// public ClickHouseColumnSettings(Encoding? stringEncoding = null, IClickHouseEnumConverter? enumConverter = null, Type? columnType = null) { StringEncoding = stringEncoding; EnumConverter = enumConverter; ColumnType = columnType; } internal ITypeDispatcher? GetColumnTypeDispatcher() { if (ColumnType == null) return null; return _columnTypeDispatcher ??= TypeDispatcher.Create(ColumnType); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseColumnWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Provides a way of writing set of columns to a ClickHouse database. This class cannot be inherited. /// public sealed class ClickHouseColumnWriter : IDisposable, IAsyncDisposable { private readonly ClickHouseTcpClient.Session _session; private readonly ClientQueryMessage _query; private readonly ReadOnlyCollection _columns; private ClickHouseColumnSettings?[]? _columnSettings; private int? _rowsPerBlock; private bool _endOfStream; /// /// Gets the number of fields (columns) in the table. /// public int FieldCount => _columns.Count; /// /// Gets the value indicating whether the writer is closed. /// /// if the reader is closed; otherwise . public bool IsClosed => _session.IsDisposed || _session.IsFailed; /// /// Gets or sets the maximal number of rows in a single block of data. /// /// The maximal number of rows in a single block of data. if the size of the block is not limited. public int? MaxBlockSize { get => _rowsPerBlock; set { if (value <= 0) throw new ArgumentException("A number of rows in a block must be greater than zero."); _rowsPerBlock = value; } } /// /// Gets the query execution progress reported by the server. /// public ClickHouseQueryExecutionProgress ExecutionProgress { get; private set; } internal ClickHouseColumnWriter(ClickHouseTcpClient.Session session, ClientQueryMessage query, ReadOnlyCollection columns) { _session = session ?? throw new ArgumentNullException(nameof(session)); _query = query ?? throw new ArgumentNullException(nameof(query)); _columns = columns; if (columns.Count <= 100) MaxBlockSize = 8000; else if (columns.Count >= 1000) MaxBlockSize = 800; else MaxBlockSize = 8800 - 8 * columns.Count; } internal static async ValueTask ReadTableMetadata(ClickHouseTcpClient.Session session, string queryText, bool async, CancellationToken cancellationToken) { var msg = await session.ReadMessage(async, cancellationToken); switch (msg.MessageCode) { case ServerMessageCode.Error: throw ((ServerErrorMessage)msg).Exception.CopyWithQuery(queryText); case ServerMessageCode.TableColumns: break; default: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message. Received the message of type {msg.MessageCode}."); } msg = await session.ReadMessage(async, cancellationToken); ClickHouseTable data; switch (msg.MessageCode) { case ServerMessageCode.Error: throw ((ServerErrorMessage)msg).Exception.CopyWithQuery(queryText); case ServerMessageCode.Data: data = await session.ReadTable((ServerDataMessage)msg, null, async, cancellationToken); break; default: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message. Received the message of type {msg.MessageCode}."); } return data; } /// public void ConfigureColumn(string name, ClickHouseColumnSettings columnSettings) { var index = GetOrdinal(name); if (index < 0) throw new ArgumentException($"A column with the name \"{name}\" not found.", nameof(name)); ConfigureColumn(index, columnSettings); } /// public void ConfigureColumn(int ordinal, ClickHouseColumnSettings columnSettings) { if (_columnSettings == null) _columnSettings = new ClickHouseColumnSettings?[_columns.Count]; _columnSettings[ordinal] = columnSettings; } /// public void ConfigureColumnWriter(ClickHouseColumnSettings columnSettings) { if (_columnSettings == null) _columnSettings = new ClickHouseColumnSettings?[_columns.Count]; for (int i = 0; i < _columns.Count; i++) _columnSettings[i] = columnSettings; } /// public IClickHouseTypeInfo GetFieldTypeInfo(int ordinal) { return _columns[ordinal].TypeInfo; } /// public string GetName(int ordinal) { return _columns[ordinal].Name; } /// public string GetDataTypeName(int ordinal) { return _columns[ordinal].TypeInfo.ComplexTypeName; } /// public Type GetFieldType(int ordinal) { // This method should implement the same logic as ClickHouseDataReader.GetFieldType var type = _columnSettings?[ordinal]?.ColumnType; type ??= _columns[ordinal].TypeInfo.GetFieldType(); return Nullable.GetUnderlyingType(type) ?? type; } /// public int GetOrdinal(string name) { if (name == null) throw new ArgumentNullException(nameof(name)); return CommonUtils.GetColumnIndex(_columns, name); } /// /// Writes a single row to the table. /// /// The list of column values. /// Please note that the method always commits a transaction. No subsequent call of is required. public void WriteRow(params object?[] values) { TaskHelper.WaitNonAsyncTask(WriteRow(values, commit: true, async: false, CancellationToken.None)); } /// /// Writes a single row to the table. /// /// The list of column values. /// A representing asyncronous operation. /// Please note that the method always commits a transaction. No subsequent call of is required. public void WriteRow(IReadOnlyCollection values) { TaskHelper.WaitNonAsyncTask(WriteRow(values, commit: true, async: false, CancellationToken.None)); } /// /// Writes a single row to the table. /// /// The list of column values. /// /// If , commits the transaction immediately after writing a row (the same mode as ). /// If , leaves the transaction open (the same mode as ). /// /// A representing asyncronous operation. public void WriteRow(IReadOnlyCollection values, bool commit) { TaskHelper.WaitNonAsyncTask(WriteRow(values, commit, async: false, CancellationToken.None)); } /// /// Asyncronously writes a single row to the table. /// /// The list of column values. /// A representing asyncronous operation. /// Please note that the method always commits a transaction. No subsequent call of is required. public async Task WriteRowAsync(IReadOnlyCollection values) { await WriteRow(values, commit: true, async: true, CancellationToken.None); } /// /// Asyncronously writes a single row to the table. /// /// The list of column values. /// /// If , commits the transaction immediately after writing a row (the same mode as ). /// If , leaves the transaction open (the same mode as ). /// /// A representing asyncronous operation. public async Task WriteRowAsync(IReadOnlyCollection values, bool commit) { await WriteRow(values, commit, async: true, CancellationToken.None); } /// /// Asyncronously writes a single row to the table. /// /// The list of column values. /// The cancellation instruction. /// A representing asyncronous operation. /// Please note that the method always commits a transaction. No subsequent call of is required. public async Task WriteRowAsync(IReadOnlyCollection values, CancellationToken cancellationToken) { await WriteRow(values, commit: true, async: true, cancellationToken); } /// /// Asyncronously writes a single row to the table. /// /// The list of column values. /// /// If , commits the transaction immediately after writing a row (the same mode as ). /// If , leaves the transaction open (the same mode as ). /// /// The cancellation instruction. /// A representing asyncronous operation. public async Task WriteRowAsync(IReadOnlyCollection values, bool commit, CancellationToken cancellationToken) { await WriteRow(values, commit, async: true, cancellationToken); } private async ValueTask WriteRow(IReadOnlyCollection values, bool commit, bool async, CancellationToken cancellationToken) { if (values == null) throw new ArgumentNullException(nameof(values)); if (values.Count != _columns.Count) throw new ArgumentException("The number of values must be equal to the number of columns."); var columnWriters = new List(_columns.Count); foreach (var value in values) { int i = columnWriters.Count; var columnInfo = _columns[i]; var settings = _columnSettings?[i]; if (settings?.ColumnType == typeof(object)) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidColumnSettings, $"Type \"{settings.ColumnType}\" should not be used as a type of a column. This type is defined in column settings of the column \"{columnInfo.Name}\" (position {i})."); } ITypeDispatcher? typeDispatcher; SingleRowColumnWriterDispatcher dispatcher; if (value != null && !(value is DBNull)) { dispatcher = new SingleRowColumnWriterDispatcher(value, columnInfo, _columnSettings?[i]); var valueType = value.GetType(); if (settings?.ColumnType != null) { if (!settings.ColumnType.IsAssignableFrom(valueType)) { throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"The value of the row at the position {i} (column \"{columnInfo.Name}\") can't be converted to the type \"{settings.ColumnType}\". This type is defined in column settings."); } typeDispatcher = settings.GetColumnTypeDispatcher(); Debug.Assert(typeDispatcher != null); } else { typeDispatcher = TypeDispatcher.Create(valueType); } } else if (columnInfo.TypeInfo.TypeName != "Nullable") { throw new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" at the position {i} doesn't support nulls."); } else { dispatcher = new SingleRowColumnWriterDispatcher(null, columnInfo, _columnSettings?[i]); if (settings?.ColumnType != null) { if (settings.ColumnType.IsValueType && Nullable.GetUnderlyingType(settings.ColumnType) == null) { throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"The value of the row at the position {i} (column \"{columnInfo.Name}\") is null. But the type of this column defined in the settings (\"{settings.ColumnType}\") doesn't allow nulls."); } typeDispatcher = settings.GetColumnTypeDispatcher(); Debug.Assert(typeDispatcher != null); } else { var fieldType = columnInfo.TypeInfo.GetFieldType(); typeDispatcher = TypeDispatcher.Create(fieldType); } } IClickHouseColumnWriter columnWriter; try { columnWriter = typeDispatcher.Dispatch(dispatcher); } catch (ClickHouseException ex) { throw new ClickHouseException(ex.ErrorCode, $"Column \"{columnInfo.Name}\" (position {i}): {ex.Message}", ex); } columnWriters.Add(columnWriter); } var table = new ClickHouseTableWriter(string.Empty, 1, columnWriters); await SendTable(table, commit, async, cancellationToken); } /// /// Writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// or /// . /// /// The object that provides access to columns by their names. /// The number of rows in columns. public void WriteTable(IReadOnlyDictionary columns, int rowCount) { TaskHelper.WaitNonAsyncTask(WriteTable(columns, rowCount, ClickHouseTransactionMode.Default, false, CancellationToken.None)); } /// /// Writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// or /// . /// /// The object that provides access to columns by their names. /// The number of rows in columns. /// The mode of sending write confirmations to the server.See for details. public void WriteTable(IReadOnlyDictionary columns, int rowCount, ClickHouseTransactionMode transactionMode) { TaskHelper.WaitNonAsyncTask(WriteTable(columns, rowCount, transactionMode, false, CancellationToken.None)); } /// /// Writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// or /// . /// /// The list of columns. /// The number of rows in columns. public void WriteTable(IReadOnlyList columns, int rowCount) { TaskHelper.WaitNonAsyncTask(WriteTable(columns, rowCount, ClickHouseTransactionMode.Default, false, CancellationToken.None)); } /// /// Writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// or /// . /// /// The list of columns. /// The number of rows in columns. /// The mode of sending write confirmations to the server.See for details. public void WriteTable(IReadOnlyList columns, int rowCount, ClickHouseTransactionMode transactionMode) { TaskHelper.WaitNonAsyncTask(WriteTable(columns, rowCount, transactionMode, false, CancellationToken.None)); } /// /// Asyncronously writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// , /// or /// . /// /// The object that provides access to columns by their names. /// The number of rows in columns. /// The cancellation instruction. /// A representing asyncronous operation. public async Task WriteTableAsync(IReadOnlyDictionary columns, int rowCount, CancellationToken cancellationToken) { await WriteTable(columns, rowCount, ClickHouseTransactionMode.Default, true, cancellationToken); } /// /// Asyncronously writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// , /// or /// . /// /// The object that provides access to columns by their names. /// The number of rows in columns. /// The mode of sending write confirmations to the server.See for details. /// The cancellation instruction. /// A representing asyncronous operation. public async Task WriteTableAsync(IReadOnlyDictionary columns, int rowCount, ClickHouseTransactionMode transactionMode, CancellationToken cancellationToken) { await WriteTable(columns, rowCount, transactionMode, true, cancellationToken); } /// /// Asyncronously writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// , /// or /// . /// /// The list of columns. /// The number of rows in columns. /// The cancellation instruction. /// A representing asyncronous operation. public async Task WriteTableAsync(IReadOnlyList columns, int rowCount, CancellationToken cancellationToken) { await WriteTable(columns, rowCount, ClickHouseTransactionMode.Default, true, cancellationToken); } /// /// Asyncronously writes the specified columns to the table. ///
/// Each column must be an object implementing one of the interfaces: /// , /// , /// , /// or /// . /// /// The list of columns. /// The number of rows in columns. /// The mode of sending write confirmations to the server.See for details. /// The cancellation instruction. /// A representing asyncronous operation. public async Task WriteTableAsync(IReadOnlyList columns, int rowCount, ClickHouseTransactionMode transactionMode, CancellationToken cancellationToken) { await WriteTable(columns, rowCount, transactionMode, true, cancellationToken); } private async ValueTask WriteTable(IReadOnlyDictionary columns, int rowCount, ClickHouseTransactionMode mode, bool async, CancellationToken cancellationToken) { if (columns == null) throw new ArgumentNullException(nameof(columns)); var list = new List(_columns.Count); foreach (var columnInfo in _columns) { if (columns.TryGetValue(columnInfo.Name, out var column)) list.Add(column); else list.Add(null); } await WriteTable(list, rowCount, mode, async, cancellationToken); } private async ValueTask WriteTable(IReadOnlyList columns, int rowCount, ClickHouseTransactionMode mode, bool async, CancellationToken cancellationToken) { if (columns == null) throw new ArgumentNullException(nameof(columns)); if (columns.Count != _columns.Count) throw new ArgumentException("The number of columns for writing must be equal to the number of columns in the table.", nameof(columns)); if (rowCount < 0) throw new ArgumentOutOfRangeException(nameof(rowCount)); if (rowCount == 0) throw new ArgumentException("The number of rows must be greater than zero.", nameof(rowCount)); if (IsClosed) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The writer is closed."); var writerFactories = new List(_columns.Count); for (int i = 0; i < _columns.Count; i++) { var factory = await CreateColumnWriterFactory(_columns[i], columns[i], i, rowCount, _columnSettings?[i], async, cancellationToken); writerFactories.Add(factory); } int offset; var blockSize = MaxBlockSize ?? rowCount; bool commitBlock = mode == ClickHouseTransactionMode.Block; for (offset = 0; offset + blockSize < rowCount; offset += blockSize) { var table = new ClickHouseTableWriter(string.Empty, blockSize, writerFactories.Select(w => w.Create(offset, blockSize))); await SendTable(table, commitBlock, async, cancellationToken); } var finalBlockSize = rowCount - offset; var finalTable = new ClickHouseTableWriter(string.Empty, finalBlockSize, writerFactories.Select(w => w.Create(offset, finalBlockSize))); bool commit = commitBlock || mode == ClickHouseTransactionMode.Default || mode == ClickHouseTransactionMode.Auto; await SendTable(finalTable, commit, async, cancellationToken); } private async ValueTask SendTable(ClickHouseTableWriter table, bool commit, bool async, CancellationToken cancellationToken) { if (_endOfStream) await RepeatQuery(async, cancellationToken); try { await _session.SendTable(table, async, cancellationToken); if (commit) await EndWrite(TerminationMode.Confirm, closeSession: false, async, cancellationToken); } catch (ClickHouseHandledException) { throw; } catch (Exception ex) { var aggrEx = await _session.SetFailed(ex, false, async); if (aggrEx != null) throw aggrEx; throw; } } private async ValueTask RepeatQuery(bool async, CancellationToken cancellationToken) { ClickHouseTable data; try { await _session.SendQuery(_query, async, cancellationToken); data = await ReadTableMetadata(_session, _query.Query, async, cancellationToken); _endOfStream = false; } catch (ClickHouseServerException) { await _session.Dispose(async); throw; } catch (ClickHouseHandledException) { await _session.Dispose(async); throw; } catch (Exception ex) { var aggrEx = await _session.SetFailed(ex, false, async); if (aggrEx != null) throw aggrEx; throw; } try { // Repeating the query is almost the same as opening a new independent column writer. So we must check that the structure of the table wasn't changed. var newColumns = data.Header.Columns; if (newColumns.Count != _columns.Count) throw new ClickHouseException(ClickHouseErrorCodes.TableModified, "The number of columns returned by the query has changed."); for (int i = 0; i < data.Columns.Count; i++) { var newCol = newColumns[i]; var origCol = _columns[i]; if (!string.Equals(origCol.Name, newCol.Name, StringComparison.Ordinal)) throw new ClickHouseException(ClickHouseErrorCodes.TableModified, $"Unexpected column \"{newCol.Name}\" at the position {i}. Expected \"{origCol.Name}\"."); if (!string.Equals(origCol.TypeInfo.ComplexTypeName, newCol.TypeInfo.ComplexTypeName, StringComparison.Ordinal)) throw new ClickHouseException(ClickHouseErrorCodes.TableModified, $"The type of the column \"{origCol.Name}\" has changed from \"{origCol.TypeInfo.ComplexTypeName}\" to \"{newCol.TypeInfo.ComplexTypeName}\" between queries."); } } catch (Exception ex) { try { await EndWrite(TerminationMode.Cancel, closeSession: true, async, cancellationToken); } catch (Exception cancellationEx) { throw new AggregateException(ex, cancellationEx); } var hEx = ClickHouseHandledException.Wrap(ex); if (ReferenceEquals(hEx, ex)) throw; throw hEx; } } /// /// Notifies the server that the transaction should be commited. /// This method acts similar to , but it doesn't close the writer. /// /// A subsequent writing operation will send a new INSERT query to the server. public void Commit() { TaskHelper.WaitNonAsyncTask(EndWrite(TerminationMode.Confirm, closeSession: false, async: false, CancellationToken.None)); } /// /// Asyncronously notifies the server that the transaction should be commited. /// This method acts similar to , but it doesn't close the writer. /// /// The cancellation instruction. /// A representing asyncronous operation. /// A subsequent writing operation will send a new INSERT query to the server. public async Task CommitAsync(CancellationToken cancellationToken) { await EndWrite(TerminationMode.Confirm, closeSession: false, async: true, cancellationToken); } /// /// Closes the writer and releases all resources associated with it. /// public void EndWrite() { TaskHelper.WaitNonAsyncTask(EndWrite(TerminationMode.Confirm, closeSession: true, false, CancellationToken.None)); } /// /// Notifies the server that non-commited rows shoud be discarded. This method takes an effect /// only if the pervious operation was made in the mode. /// public void Rollback() { TaskHelper.WaitNonAsyncTask(EndWrite(TerminationMode.Cancel, closeSession: false, async: false, CancellationToken.None)); } /// /// Asyncronously notifies the server that non-commited rows shoud be discarded. This method takes an effect /// only if the pervious operation was made in the mode. /// /// The cancellation instruction. /// A representing asyncronous operation. /// A subsequent writing operation will send a new INSERT query to the server. public async Task RollbackAsync(CancellationToken cancellationToken) { await EndWrite(TerminationMode.Cancel, closeSession: false, async: true, cancellationToken); } /// /// Asyncronously closes the writer and releases all resources associated with it. /// /// The cancellation instruction. /// A representing asyncronous operation. public async Task EndWriteAsync(CancellationToken cancellationToken) { await EndWrite(TerminationMode.Confirm, closeSession: true, true, cancellationToken); } private async ValueTask EndWrite(TerminationMode mode, bool closeSession, bool async, CancellationToken cancellationToken) { // If the writer is dispesed the session should also be disposed Debug.Assert(closeSession || mode != TerminationMode.Dispose); if (IsClosed) return; if (_endOfStream) { if (closeSession) await _session.Dispose(async); return; } try { switch (mode) { case TerminationMode.None: break; case TerminationMode.Cancel: case TerminationMode.Dispose: await _session.SendCancel(async); break; case TerminationMode.Confirm: await _session.SendTable(ClickHouseEmptyTableWriter.Instance, async, cancellationToken); break; default: Debug.Fail($"Unexpected termination mode: {mode}."); break; } bool isProfileEvents; do { isProfileEvents = false; var message = await _session.ReadMessage(async, CancellationToken.None); switch (message.MessageCode) { case ServerMessageCode.EndOfStream: if (closeSession) await _session.Dispose(async); _endOfStream = true; break; case ServerMessageCode.Error: // Connection state can't be resotred if the server raised an exception. // This error is probably caused by the wrong formatted data. var exception = ((ServerErrorMessage)message).Exception; if (mode == TerminationMode.Dispose) { await _session.SetFailed(exception, false, async); break; } throw exception; case ServerMessageCode.ProfileEvents: isProfileEvents = true; var profileEventsMessage = (ServerDataMessage)message; await _session.SkipTable(profileEventsMessage, async, cancellationToken); break; case ServerMessageCode.ProfileInfo: break; case ServerMessageCode.Progress: var progressMessage = (ServerProgressMessage)message; ExecutionProgress = progressMessage.ExecutionProgress; break; default: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message: \"{message.MessageCode}\"."); } } while (isProfileEvents); } catch (ClickHouseHandledException ex) { if (mode != TerminationMode.Dispose) throw; // Connection state can't be restored await _session.SetFailed(ex.InnerException, false, async); } catch (Exception ex) { var aggrEx = await _session.SetFailed(ex, false, async); if (aggrEx != null) throw aggrEx; throw; } } /// /// Closes the writer and releases all resources associated with it. /// public void Dispose() { TaskHelper.WaitNonAsyncTask(Dispose(false)); } /// /// Asyncronously closes the writer and releases all resources associated with it. /// /// A representing asyncronous operation. public ValueTask DisposeAsync() { return Dispose(true); } private async ValueTask Dispose(bool async) { await EndWrite(TerminationMode.Dispose, closeSession: true, async, CancellationToken.None); } internal static async ValueTask CreateColumnWriterFactory(ColumnInfo columnInfo, object? column, int columnIndex, int rowCount, ClickHouseColumnSettings? settings, bool async, CancellationToken cancellationToken) { if (settings?.ColumnType == typeof(object)) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidColumnSettings, $"Type \"{settings.ColumnType}\" should not be used as a type of a column. This type is defined in column settings of the column \"{columnInfo.Name}\" (position {columnIndex})."); } if (column == null) { if (!columnInfo.TypeInfo.TypeName.StartsWith("Nullable")) throw new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" at the position {columnIndex} doesn't support nulls."); ITypeDispatcher? typeDispatcher; if (settings?.ColumnType != null) { if (settings.ColumnType.IsValueType && Nullable.GetUnderlyingType(settings.ColumnType) == null) { throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" (position {columnIndex}) contains null value. But the type of this column defined in the settings (\"{settings.ColumnType}\") doesn't allow nulls."); } typeDispatcher = settings.GetColumnTypeDispatcher(); Debug.Assert(typeDispatcher != null); } else { typeDispatcher = TypeDispatcher.Create(columnInfo.TypeInfo.GetFieldType()); } var constColumn = typeDispatcher.Dispatch(new NullColumnWriterDispatcher(columnInfo, settings, rowCount)); return constColumn; } var columnType = column.GetType(); Type? enumerable = null; Type? altEnumerable = null; Type? asyncEnumerable = null; Type? altAsyncEnumerable = null; Type? readOnlyList = null; Type? altReadOnlyList = null; Type? list = null; Type? altList = null; foreach (var ifs in columnType.GetInterfaces()) { if (!ifs.IsGenericType) continue; var ifsDefinition = ifs.GetGenericTypeDefinition(); if (ifsDefinition == typeof(IEnumerable<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altEnumerable ??= enumerable; enumerable = ifs; } else if (ifsDefinition == typeof(IAsyncEnumerable<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altAsyncEnumerable = asyncEnumerable; asyncEnumerable = ifs; } else if (ifsDefinition == typeof(IReadOnlyList<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altReadOnlyList = readOnlyList; readOnlyList = ifs; } else if (ifsDefinition == typeof(IList<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altList = list; list = ifs; } } /* * All supported interfaces (sorted by priority): * 1. IReadOnlyList * 2. IList * 3. IAsyncEnumerable (supported only in ascynronuous mode, i.e. async == true) * 4. IEnumerable * 5. IEnumerable */ var explicitTypeDispatcher = settings?.GetColumnTypeDispatcher(); IClickHouseColumnWriterFactory? columnWriter; if (explicitTypeDispatcher != null) { Debug.Assert(settings?.ColumnType != null); // The type is explicitly specified in the column settings. Either cast the column to a collection // of this type or throw an exception. columnWriter = explicitTypeDispatcher.Dispatch(new ColumnWriterDispatcher(column, columnInfo, settings, rowCount, columnIndex, async)); if (columnWriter != null) return columnWriter; if (async && typeof(IAsyncEnumerable<>).MakeGenericType(settings.ColumnType).IsAssignableFrom(columnType)) { columnWriter = await explicitTypeDispatcher.Dispatch(new AsyncColumnWriterDispatcher(column, columnInfo, settings, rowCount, columnIndex, cancellationToken)); return columnWriter; } // There is almost no chance that IEnumerable's IEnumerator returns the value of expected type if at least one of interfaces is implemented by the column's type. bool ignoreNonGenericEnumerable = readOnlyList != null || list != null || asyncEnumerable != null || enumerable != null; columnWriter = explicitTypeDispatcher.Dispatch(new ColumnWriterObjectCollectionDispatcher(column, columnInfo, settings, rowCount, columnIndex, async, ignoreNonGenericEnumerable)); if (columnWriter != null) return columnWriter; if (async && column is IAsyncEnumerable aeCol) { columnWriter = await explicitTypeDispatcher.Dispatch(new AsyncObjectColumnWriterDispatcher(aeCol, columnInfo, settings, rowCount, columnIndex, cancellationToken)); return columnWriter; } if (!async) { Type? aeInterface = typeof(IAsyncEnumerable<>).MakeGenericType(settings.ColumnType); if (!aeInterface.IsAssignableFrom(settings.ColumnType)) { aeInterface = column is IAsyncEnumerable ? typeof(IAsyncEnumerable) : null; } if (aeInterface != null) { throw new ClickHouseException( ClickHouseErrorCodes.NotSupportedInSyncronousMode, $"The column \"{columnInfo.Name}\" at the position {columnIndex} implements interface \"{aeInterface}\". Call async method \"{nameof(WriteTableAsync)}\"."); } } throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" at the position {columnIndex} is not a collection of type \"{settings.ColumnType}\". This type is defined in the column's settings."); } // Trying to extract an actual type of column's items from an interface implemented by this column. Type dispatchedElementType; if (readOnlyList != null) { if (altReadOnlyList != null) throw CreateInterfaceAmbiguousException(readOnlyList, altReadOnlyList, columnInfo.Name, columnIndex); dispatchedElementType = readOnlyList.GetGenericArguments()[0]; } else if (list != null) { if (altList != null) throw CreateInterfaceAmbiguousException(list, altList, columnInfo.Name, columnIndex); dispatchedElementType = list.GetGenericArguments()[0]; } else { if (async && asyncEnumerable != null) { if (altAsyncEnumerable != null) throw CreateInterfaceAmbiguousException(asyncEnumerable, altAsyncEnumerable, columnInfo.Name, columnIndex); var genericArg = asyncEnumerable.GetGenericArguments()[0]; var asyncDispatcher = new AsyncColumnWriterDispatcher(column, columnInfo, settings, rowCount, columnIndex, cancellationToken); var asyncColumn = await TypeDispatcher.Dispatch(genericArg, asyncDispatcher); return asyncColumn; } if (enumerable != null) { if (altEnumerable != null) throw CreateInterfaceAmbiguousException(enumerable, altEnumerable, columnInfo.Name, columnIndex); dispatchedElementType = enumerable.GetGenericArguments()[0]; } else { // There is still hope that the column implements one of suported interfaces with typeof(T) == typeof(object). // In this case assume that the type of the table's field is equal to the type of the column. dispatchedElementType = columnInfo.TypeInfo.GetFieldType(); var typeDispatcher = TypeDispatcher.Create(dispatchedElementType); var objDispatcher = new ColumnWriterObjectCollectionDispatcher(column, columnInfo, settings, rowCount, columnIndex, async); var objColumnWriter = typeDispatcher.Dispatch(objDispatcher); if (async && objColumnWriter == null && column is IAsyncEnumerable aeCol) { var asyncDispatcher = new AsyncObjectColumnWriterDispatcher(aeCol, columnInfo, settings, rowCount, columnIndex, cancellationToken); objColumnWriter = await typeDispatcher.Dispatch(asyncDispatcher); } if (objColumnWriter == null) { if (!async && (asyncEnumerable != null || column is IAsyncEnumerable)) { var aeInterface = asyncEnumerable ?? typeof(IAsyncEnumerable); throw new ClickHouseException( ClickHouseErrorCodes.NotSupportedInSyncronousMode, $"The column \"{columnInfo.Name}\" at the position {columnIndex} implements interface \"{aeInterface}\". Call async method \"{nameof(WriteTableAsync)}\"."); } throw new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" at the position {columnIndex} is not a collection."); } return objColumnWriter; } } var dispatcher = new ColumnWriterDispatcher(column, columnInfo, settings, rowCount, columnIndex, false); columnWriter = TypeDispatcher.Dispatch(dispatchedElementType, dispatcher); if (columnWriter == null) throw new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"The column \"{columnInfo.Name}\" at the position {columnIndex} is not a collection."); return columnWriter; } private static ClickHouseException CreateInterfaceAmbiguousException(Type itf, Type altItf, string columnName, int columnIndex) { return new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"A type of the column \"{columnName}\" at the position {columnIndex} is ambiguous. The column implements interfaces \"{itf}\" and \"{altItf}\"."); } private class ColumnWriterFactory : IClickHouseColumnWriterFactory { private readonly ColumnInfo _columnInfo; private readonly IReadOnlyList _rows; private readonly ClickHouseColumnSettings? _columnSettings; public ColumnWriterFactory(ColumnInfo columnInfo, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { _columnInfo = columnInfo; _rows = rows; _columnSettings = columnSettings; } public IClickHouseColumnWriter Create(int offset, int length) { var slice = _rows.Slice(offset, length); return _columnInfo.TypeInfo.CreateColumnWriter(_columnInfo.Name, slice, _columnSettings); } } private class NullColumnWriterDispatcher : ITypeDispatcher { private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; private readonly int _rowCount; public NullColumnWriterDispatcher(ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings, int rowCount) { _columnInfo = columnInfo; _columnSettings = columnSettings; _rowCount = rowCount; } public IClickHouseColumnWriterFactory Dispatch() { var rows = new ConstantReadOnlyList(default, _rowCount); return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } } private class AsyncColumnWriterDispatcher : ITypeDispatcher> { private readonly object _asyncEnumerable; private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; private readonly int _rowCount; private readonly int _columnIndex; private readonly CancellationToken _cancellationToken; public AsyncColumnWriterDispatcher( object asyncEnumerable, ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings, int rowCount, int columnIndex, CancellationToken cancellationToken) { _asyncEnumerable = asyncEnumerable; _columnInfo = columnInfo; _columnSettings = columnSettings; _rowCount = rowCount; _columnIndex = columnIndex; _cancellationToken = cancellationToken; } public async Task Dispatch() { if (_rowCount == 0) return new ColumnWriterFactory(_columnInfo, Array.Empty(), _columnSettings); ConfiguredCancelableAsyncEnumerable.Enumerator enumerator = default; bool disposeEnumerator = false; try { enumerator = ((IAsyncEnumerable)_asyncEnumerable).WithCancellation(_cancellationToken).GetAsyncEnumerator(); disposeEnumerator = true; var rows = new T[_rowCount]; for (int i = 0; i < _rowCount; i++) { if (!await enumerator.MoveNextAsync()) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {i} row(s), but the required number of rows is {_rowCount}."); } rows[i] = enumerator.Current; } return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } finally { if (disposeEnumerator) await enumerator.DisposeAsync(); } } } private class AsyncObjectColumnWriterDispatcher : ITypeDispatcher> { private readonly IAsyncEnumerable _asyncEnumerable; private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; private readonly int _rowCount; private readonly int _columnIndex; private readonly CancellationToken _cancellationToken; public AsyncObjectColumnWriterDispatcher( IAsyncEnumerable asyncEnumerable, ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings, int rowCount, int columnIndex, CancellationToken cancellationToken) { _asyncEnumerable = asyncEnumerable; _columnInfo = columnInfo; _columnSettings = columnSettings; _rowCount = rowCount; _columnIndex = columnIndex; _cancellationToken = cancellationToken; } public async Task Dispatch() { if (_rowCount == 0) return new ColumnWriterFactory(_columnInfo, Array.Empty(), _columnSettings); ConfiguredCancelableAsyncEnumerable.Enumerator enumerator = default; bool disposeEnumerator = false; try { enumerator = _asyncEnumerable.WithCancellation(_cancellationToken).GetAsyncEnumerator(); disposeEnumerator = true; var rows = new T[_rowCount]; for (int i = 0; i < _rowCount; i++) { if (!await enumerator.MoveNextAsync()) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {i} row(s), but the required number of rows is {_rowCount}."); } rows[i] = ColumnWriterObjectCollectionDispatcher.CastTo(enumerator.Current); } return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } finally { if (disposeEnumerator) await enumerator.DisposeAsync(); } } } private class ColumnWriterDispatcher : ITypeDispatcher { private readonly object _collection; private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; private readonly int _rowCount; private readonly int _columnIndex; private readonly bool _checkAsyncEnumerable; public ColumnWriterDispatcher( object collection, ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings, int rowCount, int columnIndex, bool checkAsyncEnumerable) { _collection = collection; _columnInfo = columnInfo; _columnSettings = columnSettings; _rowCount = rowCount; _columnIndex = columnIndex; _checkAsyncEnumerable = checkAsyncEnumerable; } public IClickHouseColumnWriterFactory? Dispatch() { if (_collection is IReadOnlyList readOnlyList) { if (readOnlyList.Count < _rowCount) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {readOnlyList.Count} row(s), but the required number of rows is {_rowCount}."); } if (readOnlyList.Count > _rowCount) readOnlyList = readOnlyList.Slice(0, _rowCount); return new ColumnWriterFactory(_columnInfo, readOnlyList, _columnSettings); } if (_collection is IList list) { if (list.Count < _rowCount) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {list.Count} row(s), but the required number of rows is {_rowCount}."); } var listSpan = list.Slice(0, _rowCount); return new ColumnWriterFactory(_columnInfo, listSpan, _columnSettings); } if (_checkAsyncEnumerable && _collection is IAsyncEnumerable) { // Should be handled in async mode return null; } if (_collection is IEnumerable genericEnumerable) { using var enumerator = genericEnumerable.GetEnumerator(); T[] rows = new T[_rowCount]; for (int i = 0; i < _rowCount; i++) { if (!enumerator.MoveNext()) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {i} row(s), but the required number of rows is {_rowCount}."); } rows[i] = enumerator.Current; } return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } return null; } } private class ColumnWriterObjectCollectionDispatcher : ITypeDispatcher { private readonly object _collection; private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; private readonly int _rowCount; private readonly int _columnIndex; private readonly bool _checkAsyncEnumerable; private readonly bool _ignoreNonGenericEnumerable; public ColumnWriterObjectCollectionDispatcher( object collection, ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings, int rowCount, int columnIndex, bool checkAsyncEnumerable, bool ignoreNonGenericEnumerable = false) { _collection = collection; _columnInfo = columnInfo; _columnSettings = columnSettings; _rowCount = rowCount; _columnIndex = columnIndex; _checkAsyncEnumerable = checkAsyncEnumerable; _ignoreNonGenericEnumerable = ignoreNonGenericEnumerable; } public IClickHouseColumnWriterFactory? Dispatch() { if (_collection is IReadOnlyList readOnlyList) { if (readOnlyList.Count < _rowCount) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {readOnlyList.Count} row(s), but the required number of rows is {_rowCount}."); } if (readOnlyList.Count > _rowCount) readOnlyList = readOnlyList.Slice(0, _rowCount); } else if (_collection is IList list) { if (list.Count < _rowCount) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {list.Count} row(s), but the required number of rows is {_rowCount}."); } readOnlyList = list.Slice(0, _rowCount); } else if (_checkAsyncEnumerable && _collection is IAsyncEnumerable) { // Should be handled in async mode return null; } else if (_collection is IEnumerable genericEnumerable) { using var enumerator = genericEnumerable.GetEnumerator(); T[] rows = new T[_rowCount]; for (int i = 0; i < _rowCount; i++) { if (!enumerator.MoveNext()) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {i} row(s), but the required number of rows is {_rowCount}."); } rows[i] = CastTo(enumerator.Current); } return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } else if (!_ignoreNonGenericEnumerable && _collection is IEnumerable enumerable) { IEnumerator? enumerator = null; try { enumerator = enumerable.GetEnumerator(); T[] rows = new T[_rowCount]; for (int i = 0; i < _rowCount; i++) { if (!enumerator.MoveNext()) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidRowCount, $"The column \"{_columnInfo.Name}\" at the position {_columnIndex} has only {i} row(s), but the required number of rows is {_rowCount}."); } rows[i] = CastTo(enumerator.Current); } return new ColumnWriterFactory(_columnInfo, rows, _columnSettings); } finally { (enumerator as IDisposable)?.Dispose(); } } else { // An object is not a collection return null; } var mappedList = readOnlyList.Map(CastTo); return new ColumnWriterFactory(_columnInfo, mappedList, _columnSettings); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static T CastTo(object? value) { // T may be nullable but there is no way to declare T? if (value == DBNull.Value) return (T)(default(T) is null ? null! : value); return (T)value!; } } private class SingleRowColumnWriterDispatcher : ITypeDispatcher { [AllowNull] private readonly object _value; private readonly ColumnInfo _columnInfo; private readonly ClickHouseColumnSettings? _columnSettings; public SingleRowColumnWriterDispatcher(object? value, ColumnInfo columnInfo, ClickHouseColumnSettings? columnSettings) { _value = value; _columnInfo = columnInfo; _columnSettings = columnSettings; } public IClickHouseColumnWriter Dispatch() { var rows = new ConstantReadOnlyList((T) _value, 1); return _columnInfo.TypeInfo.CreateColumnWriter(_columnInfo.Name, rows, _columnSettings); } } private enum TerminationMode { /// /// Send nothing and wait for EndOfStream /// None = 0, /// /// Send Cancel and wait for EndOfStream /// Cancel = 1, /// /// Send confirmation message (completely empty talbe) and wait for EndOfStream /// Confirm = 2, /// /// Send Cancel and then release all resources associated with the writer /// Dispose = 3 } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseCommand.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Represents an SQL statement to execute against a ClickHouse database. This class cannot be inherited. /// public sealed class ClickHouseCommand : DbCommand { private string? _commandText; private TimeSpan? _commandTimeout; /// /// Gets or sets the SQL statement to execute at the data source. /// [AllowNull] public override string CommandText { get => _commandText ?? string.Empty; set => _commandText = value; } /// /// Gets or sets the wait time (in seconds) before terminating the attempt to execute a command and generating an error. /// public override int CommandTimeout { get => (int)CommandTimeoutSpan.TotalSeconds; set => CommandTimeoutSpan = TimeSpan.FromSeconds(value); } /// /// Gets or sets the wait time before terminating the attempt to execute a command and generating an error. /// public TimeSpan CommandTimeoutSpan { get => GetCommandTimeout(Connection); set => _commandTimeout = value; } /// /// Gets the sets type of the command. The only supported type is . /// /// The value . /// The type set is not . public override CommandType CommandType { get => CommandType.Text; set { if (value != CommandType.Text) throw new NotSupportedException($"The type of the command \"{value}\" is not supported."); } } /// /// Gets or sets how command results are applied to the when used by the Update method of the . /// The value of this property is ignored by the command and therefore doesn't affect it's behavior. /// /// One of enumeration values that indicates how command results are applied. The default value is . public override UpdateRowSource UpdatedRowSource { get; set; } /// /// Gets or sets the used by this command. /// public new ClickHouseConnection? Connection { get; set; } /// protected override DbConnection? DbConnection { get => Connection; set => Connection = (ClickHouseConnection?) value; } /// /// Gets the . /// /// The parameters of the SQL statement. The default is an empty collection. public new ClickHouseParameterCollection Parameters { get; } = new ClickHouseParameterCollection(); /// protected sealed override DbParameterCollection DbParameterCollection => Parameters; /// /// Gets the . /// /// /// The tables which should be sent along with the query. The default is an empty collection. /// public ClickHouseTableProviderCollection TableProviders { get; } = new ClickHouseTableProviderCollection(); /// /// Gets or sets the transaction within which the command executes. Always returns null. /// /// null /// The value set is not null. protected override DbTransaction? DbTransaction { get => null; set { if (value != null) throw new NotSupportedException($"{nameof(DbTransaction)} is read only.'"); } } /// /// Gets or sets a value indicating whether the command object should be visible in a customized interface control. /// /// , if the command object should be visible in a control; otherwise . The default is . public override bool DesignTimeVisible { get; set; } = true; /// /// Gets or sets value indicating whether the query should be executed with an explicitly defined values of the property 'extremes'. /// public bool? Extremes { get; set; } /// /// Gets or sets a value indicating whether profile events should be ignored while reading data. /// /// , if the data reader should skip profile events. , /// if the data reader should return profile events as a recordset. The default value is . public bool IgnoreProfileEvents { get; set; } = true; /// /// Gets or sets the mode of passing parameters to the query. The value of this property overrides . /// /// The mode of passing parameters to the query. The default value is . public ClickHouseParameterMode ParametersMode { get; set; } = ClickHouseParameterMode.Inherit; /// /// Optional. Gets or sets the identifier passed along with the query as the query ID. /// Usually, a query identifier is generated by the sever, setting it on the client side is not required. /// public string? QueryId { get; set; } /// /// Creates a new instance of . /// public ClickHouseCommand() { } internal ClickHouseCommand(ClickHouseConnection connection) { Connection = connection ?? throw new ArgumentNullException(nameof(connection)); } /// /// Not supported. To cancel a command execute it asyncronously with an appropriate cancellation token. /// /// Always throws . public override void Cancel() { throw new NotImplementedException(); } /// /// Executes a SQL statement against a connection object. /// /// The number of rows affected. The returned value is negative when the actual number of rows is greater than . public override int ExecuteNonQuery() { var result = TaskHelper.WaitNonAsyncTask(ExecuteNonQuery(false, CancellationToken.None)); if (result > int.MaxValue) return int.MinValue; return (int) result; } /// /// Executes a SQL statement against a connection object asyncronously. /// /// /// /// A representing the asynchronous operation. The result () is the /// number of affected rows. The result is negative when the actual number of rows is greater than . /// public override async Task ExecuteNonQueryAsync(CancellationToken cancellationToken) { var result = await ExecuteNonQuery(true, cancellationToken); if (result > int.MaxValue) return int.MinValue; return (int) result; } private async ValueTask ExecuteNonQuery(bool async, CancellationToken cancellationToken) { ClickHouseTcpClient.Session? session = null; bool cancelOnFailure = false; try { session = await OpenSession(false, async, cancellationToken); var query = await SendQuery(session, CommandBehavior.Default, async, cancellationToken); cancelOnFailure = true; (ulong read, ulong written) result = (0, 0), progress = (0, 0); while (true) { var message = await session.ReadMessage(async, cancellationToken); switch (message.MessageCode) { case ServerMessageCode.Data: case ServerMessageCode.Totals: case ServerMessageCode.Extremes: var dataMessage = (ServerDataMessage) message; var blockHeader = await session.SkipTable(dataMessage, async, cancellationToken); if (blockHeader.Columns.Count == 0 && blockHeader.RowCount == 0) { result = (result.read + progress.read, result.written + progress.written); progress = (0, 0); } continue; case ServerMessageCode.ProfileEvents: var profileEventsMessage = (ServerDataMessage)message; await session.SkipTable(profileEventsMessage, async, cancellationToken); continue; case ServerMessageCode.Error: throw ((ServerErrorMessage) message).Exception.CopyWithQuery(query); case ServerMessageCode.EndOfStream: result = (result.read + progress.read, result.written + progress.written); await session.Dispose(async); if (result.written > 0) { // INSERT command could also return the number of parsed rows. Return only the number of inserted rows. return result.written; } return result.read; case ServerMessageCode.Progress: var progressMessage = (ServerProgressMessage) message; progress = (progressMessage.ExecutionProgress.Rows, progressMessage.ExecutionProgress.WrittenRows); continue; case ServerMessageCode.ProfileInfo: continue; case ServerMessageCode.Pong: case ServerMessageCode.Hello: case ServerMessageCode.Log: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message: \"{message.MessageCode}\"."); default: throw new NotSupportedException($"Internal error. Message code \"{message.MessageCode}\" not supported."); } } } catch (ClickHouseHandledException ex) { // Exception can't be handled at this level if (session != null) { var aggrEx = await session.SetFailed(ex.InnerException, cancelOnFailure, async); if (aggrEx != null) throw aggrEx; } throw; } catch (Exception ex) { if (session != null) { var aggrEx = await session.SetFailed(ex, cancelOnFailure, async); if (aggrEx != null) throw aggrEx; } throw; } finally { if (session != null) await session.Dispose(async); } } /// /// Executes the query and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// /// The first row of the first columns in the result set or if the result set is empty. /// public override object ExecuteScalar() { return TaskHelper.WaitNonAsyncTask(ExecuteScalar(null, false, CancellationToken.None)); } /// /// Executes the query and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// Optional parameter. Settings for the first column in the result set. /// /// The first row of the first columns in the result set or if the result set is empty. /// public object ExecuteScalar(ClickHouseColumnSettings? columnSettings) { return TaskHelper.WaitNonAsyncTask(ExecuteScalar(columnSettings, false, CancellationToken.None)); } /// /// Executes the query and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// /// The first row of the first columns in the result set. /// public T ExecuteScalar() { return TaskHelper.WaitNonAsyncTask(ExecuteScalar(null, false, CancellationToken.None)); } /// /// Executes the query and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// Optional parameter. Settings for the first column in the result set. /// /// The first row of the first columns in the result set. /// public T ExecuteScalar(ClickHouseColumnSettings? columnSettings) { return TaskHelper.WaitNonAsyncTask(ExecuteScalar(columnSettings, false, CancellationToken.None)); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The cancellation instruction. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set or if the result set is empty. /// public override async Task ExecuteScalarAsync(CancellationToken cancellationToken) { return await ExecuteScalar(null, true, cancellationToken); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// Optional parameter. Settings for the first column in the result set. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set or if the result set is empty. /// public async Task ExecuteScalarAsync(ClickHouseColumnSettings? columnSettings) { return await ExecuteScalar(columnSettings, true, CancellationToken.None); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// Optional parameter. Settings for the first column in the result set. /// The cancellation instruction. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set or if the result set is empty. /// public async Task ExecuteScalarAsync(ClickHouseColumnSettings? columnSettings, CancellationToken cancellationToken) { return await ExecuteScalar(columnSettings, true, cancellationToken); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set. /// public async Task ExecuteScalarAsync() { return await ExecuteScalar(null, true, CancellationToken.None); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// The cancellation instruction. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set. /// public async Task ExecuteScalarAsync(CancellationToken cancellationToken) { return await ExecuteScalar(null, true, cancellationToken); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// Optional parameter. Settings for the first column in the result set. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set. /// public async Task ExecuteScalarAsync(ClickHouseColumnSettings? columnSettings) { return await ExecuteScalar(columnSettings, true, CancellationToken.None); } /// /// Executes the query asyncronously and returns the first column of the first row in the result set returned by the query. /// All other columns and rows are ignored. /// /// The expected type of the first column in the result set. /// Optional parameter. Settings for the first column in the result set. /// The cancellation instruction. /// /// A representing the asynchronous operation. The result () is /// the first row of the first columns in the result set. /// public async Task ExecuteScalarAsync(ClickHouseColumnSettings? columnSettings, CancellationToken cancellationToken) { return await ExecuteScalar(columnSettings, true, cancellationToken); } private async ValueTask ExecuteScalar(ClickHouseColumnSettings? columnSettings, bool async, CancellationToken cancellationToken) { var result = await ExecuteScalar(columnSettings, reader => reader.GetFieldValue(0), async, cancellationToken); return (T) result; } private ValueTask ExecuteScalar(ClickHouseColumnSettings? columnSettings, bool async, CancellationToken cancellationToken) { return ExecuteScalar(columnSettings, reader => reader.GetValue(0), async, cancellationToken); } private async ValueTask ExecuteScalar(ClickHouseColumnSettings? columnSettings, Func valueSelector, bool async, CancellationToken cancellationToken) { ClickHouseDataReader? reader = null; try { reader = await ExecuteDbDataReader(CommandBehavior.Default, true, async, cancellationToken); bool hasAnyColumn = reader.FieldCount > 0; if (!hasAnyColumn) return DBNull.Value; if (columnSettings != null) reader.ConfigureColumn(0, columnSettings); bool hasAnyRow = async ? await reader.ReadAsync(cancellationToken) : reader.Read(); if (!hasAnyRow) return DBNull.Value; if (reader.IsDBNull(0)) return DBNull.Value; var result = valueSelector(reader); if (async) await reader.CloseAsync(); else reader.Close(); return result ?? DBNull.Value; } finally { if (async) { if (reader != null) await reader.DisposeAsync(); } else { reader?.Dispose(); } } } /// /// Not supported. A preparation of the command is not implemented. /// /// Always throws . public override void Prepare() { throw new NotImplementedException(); } /// public override Task PrepareAsync(CancellationToken cancellationToken = new CancellationToken()) { throw new NotImplementedException(); } /// /// Creates a new object with the default name and adds it to the collection of parameters (). /// /// A new object. protected override DbParameter CreateDbParameter() { const string baseParamName = "param"; int i = 0; string paramName; do { paramName = string.Format(CultureInfo.InvariantCulture, "{{{0}{1}}}", baseParamName, ++i); } while (Parameters.Contains(paramName)); return new ClickHouseParameter(paramName); } /// /// Executes the query asyncronously and builds a with the default command behavior. /// /// A representing the asynchronous operation. public new async Task ExecuteReaderAsync() { return await ExecuteDbDataReader(CommandBehavior.Default, IgnoreProfileEvents, true, CancellationToken.None); } /// /// Executes the query asyncronously and builds a with the default command behavior. /// /// The cancellation instruction. /// A representing the asynchronous operation. public new async Task ExecuteReaderAsync(CancellationToken cancellationToken) { return await ExecuteDbDataReader(CommandBehavior.Default, IgnoreProfileEvents, true, cancellationToken); } /// /// Executes the query asyncronously and builds a . /// /// /// The set of flags determining the behavior of the command. /// The flag is not supported. /// The flag is ignored. /// /// A representing the asynchronous operation. public new async Task ExecuteReaderAsync(CommandBehavior behavior) { return await ExecuteDbDataReader(behavior, IgnoreProfileEvents, true, CancellationToken.None); } /// /// Executes the query asyncronously and builds a . /// /// /// The set of flags determining the behavior of the command. /// The flag is not supported. /// The flag is ignored. /// /// The cancellation instruction. /// A representing the asynchronous operation. public new async Task ExecuteReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) { return await ExecuteDbDataReader(behavior, IgnoreProfileEvents, true, cancellationToken); } /// protected override async Task ExecuteDbDataReaderAsync(CommandBehavior behavior, CancellationToken cancellationToken) { return await ExecuteDbDataReader(behavior, IgnoreProfileEvents, true, cancellationToken); } /// /// Executes the query and builds a with the default command behavior. /// /// A object. public new ClickHouseDataReader ExecuteReader() { return TaskHelper.WaitNonAsyncTask(ExecuteDbDataReader(CommandBehavior.Default, IgnoreProfileEvents, false, CancellationToken.None)); } /// /// Executes the query and builds a . /// /// /// The set of flags determining the behavior of the command. /// The flag is not supported. /// The flag is ignored. /// /// A object. public new ClickHouseDataReader ExecuteReader(CommandBehavior behavior) { return TaskHelper.WaitNonAsyncTask(ExecuteDbDataReader(behavior, IgnoreProfileEvents, false, CancellationToken.None)); } /// protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) { return TaskHelper.WaitNonAsyncTask(ExecuteDbDataReader(behavior, IgnoreProfileEvents, false, CancellationToken.None)); } private async ValueTask ExecuteDbDataReader(CommandBehavior behavior, bool ignoreProfileEvents, bool async, CancellationToken cancellationToken) { const CommandBehavior knownBehaviorFlags = CommandBehavior.CloseConnection | CommandBehavior.KeyInfo | CommandBehavior.SchemaOnly | CommandBehavior.SequentialAccess | CommandBehavior.SingleResult | CommandBehavior.SingleRow; var unknownBehaviorFlags = behavior & ~knownBehaviorFlags; if (unknownBehaviorFlags != 0) throw new ArgumentException($"Command behavior has unknown flags ({unknownBehaviorFlags}).", nameof(behavior)); if (behavior.HasFlag(CommandBehavior.KeyInfo)) { throw new ArgumentException( $"Command behavior has unsupported flag {nameof(CommandBehavior.KeyInfo)}." + Environment.NewLine + "Please, report an issue if you have any idea of how this flag should affect the result (https://github.com/Octonica/ClickHouseClient/issues).", nameof(behavior)); } ClickHouseDataReaderRowLimit rowLimit; if (behavior.HasFlag(CommandBehavior.SchemaOnly)) { if (behavior.HasFlag(CommandBehavior.SingleRow)) throw new ArgumentException($"Command behavior's flags {nameof(CommandBehavior.SchemaOnly)} and {nameof(CommandBehavior.SingleRow)} are mutualy exclusive.", nameof(behavior)); rowLimit = ClickHouseDataReaderRowLimit.Zero; } else if (behavior.HasFlag(CommandBehavior.SingleRow)) { rowLimit = ClickHouseDataReaderRowLimit.OneRow; } else if (behavior.HasFlag(CommandBehavior.SingleResult)) { rowLimit = ClickHouseDataReaderRowLimit.OneResult; } else { rowLimit = ClickHouseDataReaderRowLimit.Infinite; } ClickHouseTcpClient.Session? session = null; bool cancelOnFailure = false; try { session = await OpenSession(behavior.HasFlag(CommandBehavior.CloseConnection), async, cancellationToken); var query = await SendQuery(session, behavior, async, cancellationToken); cancelOnFailure = true; bool isProfileEvents; IServerMessage message; var executionProgress = new ClickHouseQueryExecutionProgress(); do { isProfileEvents = false; message = await session.ReadMessage(async, cancellationToken); switch (message.MessageCode) { case ServerMessageCode.Data: break; case ServerMessageCode.Error: throw ((ServerErrorMessage)message).Exception.CopyWithQuery(query); case ServerMessageCode.ProfileEvents: isProfileEvents = true; var dataMessage = (ServerDataMessage)message; await session.SkipTable(dataMessage, async, CancellationToken.None); break; case ServerMessageCode.EndOfStream: // The query was executed without errors but returned no data. Highly likely it was a DDL query. Closing the session. await session.Dispose(async); return new ClickHouseDataReader(session); case ServerMessageCode.Progress: var progressMessage = (ServerProgressMessage)message; executionProgress = progressMessage.ExecutionProgress; break; case ServerMessageCode.ProfileInfo: case ServerMessageCode.Pong: continue; case ServerMessageCode.Totals: case ServerMessageCode.Extremes: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, "Received unexpected totals or extremes before the main dataset."); default: throw new ClickHouseException(ClickHouseErrorCodes.QueryTypeMismatch, "There is no table in the server's response."); } } while (!(message is ServerDataMessage) || isProfileEvents); var firstTable = await session.ReadTable((ServerDataMessage) message, null, async, cancellationToken); if (rowLimit == ClickHouseDataReaderRowLimit.Zero) await session.SendCancel(async); return new ClickHouseDataReader(firstTable, session, executionProgress, rowLimit, ignoreProfileEvents); } catch (ClickHouseHandledException) { if (session != null) await session.Dispose(async); throw; } catch (ClickHouseServerException) { if (session != null) await session.Dispose(async); throw; } catch (Exception ex) { Exception? aggrEx = null; if (session != null) aggrEx = await session.SetFailed(ex, cancelOnFailure, async); if (aggrEx != null) throw aggrEx; throw; } } private async ValueTask SendQuery(ClickHouseTcpClient.Session session, CommandBehavior behavior, bool async, CancellationToken cancellationToken) { string commandText; List? tableWriters = null; Dictionary? parameterWriters; try { var parametersTable = $"_{Guid.NewGuid():N}"; commandText = PrepareCommandText(session.TypeInfoProvider, parametersTable, out var binaryParameters, out parameterWriters); if (binaryParameters != null && binaryParameters.Count > 0) { var paramTableWriter = CreateParameterTableWriter(session.TypeInfoProvider, parametersTable, binaryParameters); tableWriters = new List(TableProviders.Count + 1) { paramTableWriter }; } if (TableProviders.Count > 0) { tableWriters ??= new List(TableProviders.Count); foreach (var tableProvider in TableProviders) { if (tableProvider.ColumnCount == 0) continue; var tableWriter = await CreateTableWriter(tableProvider, session.TypeInfoProvider, async, cancellationToken); tableWriters.Add(tableWriter); } } } catch (Exception ex) { throw ClickHouseHandledException.Wrap(ex); } List>? setting = null; if (Extremes != null) { setting = new List>(1) {new KeyValuePair("extremes", Extremes.Value ? "1" : "0")}; } if (session.ServerInfo.Revision >= ClickHouseProtocolRevisions.MinRevisionWithSettingsSerializedAsStrings) { if (behavior.HasFlag(CommandBehavior.SchemaOnly) || behavior.HasFlag(CommandBehavior.SingleRow)) { // https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/Settings.h // This settings are hints for the server. The result may contain more than one row. setting ??= new List>(2); setting.Add(new KeyValuePair("max_result_rows", "1")); setting.Add(new KeyValuePair("result_overflow_mode", "break")); } } var messageBuilder = new ClientQueryMessage.Builder { QueryKind = QueryKind.InitialQuery, QueryId = QueryId, Query = commandText, Settings = setting, Parameters = parameterWriters }; await session.SendQuery(messageBuilder, tableWriters, async, cancellationToken); return commandText; } private async ValueTask OpenSession(bool closeConnection, bool async, CancellationToken cancellationToken) { var connection = Connection; if (connection == null) throw new InvalidOperationException("The connection is not set. The command can't be executed without a connection."); SessionResources? resources = null; try { var timeout = GetCommandTimeout(connection); CancellationTokenSource? sessionTokenSource = null; if (timeout > TimeSpan.Zero) sessionTokenSource = new CancellationTokenSource(timeout); if (closeConnection || sessionTokenSource != null) resources = new SessionResources(closeConnection ? connection : null, sessionTokenSource); return await connection.OpenSession(async, resources, sessionTokenSource?.Token ?? CancellationToken.None, cancellationToken); } catch { if (resources != null) await resources.Release(async); throw; } } private TimeSpan GetCommandTimeout(ClickHouseConnection? connection) { return _commandTimeout ?? connection?.CommandTimeSpan ?? TimeSpan.FromSeconds(ClickHouseConnectionStringBuilder.DefaultCommandTimeout); } private ClickHouseParameterMode GetParametersMode(ClickHouseConnection? connection) { var mode = ParametersMode; if (mode != ClickHouseParameterMode.Inherit) return mode; return connection?.ParametersMode ?? ClickHouseParameterMode.Default; } private IClickHouseTableWriter CreateParameterTableWriter(IClickHouseTypeInfoProvider typeInfoProvider, string tableName, HashSet parameters) { return new ClickHouseTableWriter(tableName, 1, Parameters.Where(p => parameters.Contains(p.Id)).Select(p => p.CreateParameterColumnWriter(typeInfoProvider))); } private static async ValueTask CreateTableWriter(IClickHouseTableProvider tableProvider, IClickHouseTypeInfoProvider typeInfoProvider, bool async, CancellationToken cancellationToken) { var factories = new List(tableProvider.ColumnCount); var rowCount = tableProvider.RowCount; for (int i = 0; i < tableProvider.ColumnCount; i++) { var columnDescriptor = tableProvider.GetColumnDescriptor(i); var typeInfo = typeInfoProvider.GetTypeInfo(columnDescriptor); var columnInfo = new ColumnInfo(columnDescriptor.ColumnName, typeInfo); var column = tableProvider.GetColumn(i); var factory = await ClickHouseColumnWriter.CreateColumnWriterFactory(columnInfo, column, i, rowCount, columnDescriptor.Settings, async, cancellationToken); factories.Add(factory); } return new ClickHouseTableWriter(tableProvider.TableName, rowCount, factories.Select(f => f.Create(0, rowCount))); } private string PrepareCommandText(IClickHouseTypeInfoProvider typeInfoProvider, string parametersTable, out HashSet? binaryParameters, out Dictionary? parameterWriters) { var query = CommandText; if (string.IsNullOrEmpty(query)) throw new InvalidOperationException("Command text is not defined."); var parameterPositions = GetParameterPositions(query); binaryParameters = null; parameterWriters = null; if (parameterPositions.Count == 0) return query; var inheritParameterMode = GetParametersMode(Connection); var queryStringBuilder = new StringBuilder(query.Length); for (int i = 0; i < parameterPositions.Count; i++) { var (offset, length, typeSeparatorIdx) = parameterPositions[i]; var start = i > 0 ? parameterPositions[i - 1].offset + parameterPositions[i - 1].length : 0; queryStringBuilder.Append(query, start, parameterPositions[i].offset - start); var parameterName = typeSeparatorIdx < 0 ? query.Substring(offset, length) : query.Substring(offset + 1, typeSeparatorIdx - 1); if (!Parameters.TryGetValue(parameterName, out var parameter)) throw new ClickHouseException(ClickHouseErrorCodes.QueryParameterNotFound, $"Parameter \"{parameterName}\" not found."); if (typeSeparatorIdx >= 0) queryStringBuilder.Append("(CAST("); var parameterMode = parameter.GetParameterMode(inheritParameterMode); switch (parameterMode) { case ClickHouseParameterMode.Interpolate: { var parameterWriter = parameter.CreateParameterWriter(typeInfoProvider); parameterWriter.Interpolate(queryStringBuilder.Append(' ')).Append(' '); break; } case ClickHouseParameterMode.Default: case ClickHouseParameterMode.Binary: binaryParameters ??= new HashSet(StringComparer.OrdinalIgnoreCase); binaryParameters.Add(parameter.Id); queryStringBuilder.Append("(SELECT ").Append(parametersTable).Append('.').Append(parameter.Id).Append(" FROM ").Append(parametersTable).Append(')'); break; case ClickHouseParameterMode.Serialize: { parameterWriters ??= new Dictionary(StringComparer.OrdinalIgnoreCase); if (!parameterWriters.TryGetValue(parameter.Id, out var parameterWriter)) { parameterWriter = parameter.CreateParameterWriter(typeInfoProvider); parameterWriters.Add(parameter.Id, parameterWriter); } parameterWriter.Interpolate( queryStringBuilder, typeInfoProvider, (qb, tp) => qb.Append('{').Append(parameter.Id).Append(':').Append(tp.ComplexTypeName).Append('}')); break; } default: throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Unexpected parameter mode: {parameterMode}."); } if (typeSeparatorIdx >= 0) queryStringBuilder.Append(" AS ").Append(query, offset + typeSeparatorIdx + 1, length - typeSeparatorIdx - 2).Append("))"); } var lastPartStart = parameterPositions[^1].offset + parameterPositions[^1].length; queryStringBuilder.Append(query, lastPartStart, query.Length - lastPartStart); return queryStringBuilder.ToString(); } private static List<(int offset, int length, int typeSeparatorIdx)> GetParameterPositions(string query) { // Searching parameters outside of comments, string literals and escaped identifiers // https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/query_language/syntax.md var identifierRegex = new Regex(@"^[a-zA-Z_][0-9a-zA-Z_]*(\:.+)?$"); var simpleIdentifierRegex = new Regex("^[a-zA-Z_][0-9a-zA-Z_]*"); ReadOnlySpan significantChars = stackalloc char[7] { '-', '\'', '"', '`', '/', '{' , '@' }; ReadOnlySpan querySlice = query; var parameterPositions = new List<(int offset, int length, int typeSeparatorIdx)>(); int position = 0; while (!querySlice.IsEmpty) { var idx = querySlice.IndexOfAny(significantChars); if (idx < 0) break; var ch = querySlice[idx]; position += idx + 1; querySlice = querySlice.Slice(idx + 1); if (querySlice.IsEmpty) break; switch (ch) { case '-': if (querySlice[0] != '-') break; var endOfCommentIdx = querySlice.IndexOfAny('\r', '\n'); if (endOfCommentIdx < 0) { position += querySlice.Length; querySlice = ReadOnlySpan.Empty; } else { position += endOfCommentIdx + 1; querySlice = querySlice.Slice(endOfCommentIdx + 1); } break; case '\'': case '"': case '`': var tokenLen = ClickHouseSyntaxHelper.GetQuotedTokenLength(((ReadOnlySpan) query).Slice(position - 1), ch); if (tokenLen < 0) { position += querySlice.Length; querySlice = ReadOnlySpan.Empty; } else { position += tokenLen - 1; querySlice = querySlice.Slice(tokenLen - 1); } break; case '/': if (querySlice[0] != '*') break; ++position; querySlice = querySlice.Slice(1); while (!querySlice.IsEmpty) { var endOfMultilineCommentIdx = querySlice.IndexOf('*'); if (endOfMultilineCommentIdx < 0) { position += querySlice.Length; querySlice = ReadOnlySpan.Empty; } else { position += endOfMultilineCommentIdx + 1; querySlice = querySlice.Slice(endOfMultilineCommentIdx + 1); if (querySlice.Length > 0 && querySlice[0] == '/') { ++position; querySlice = querySlice.Slice(1); break; } } } break; case '{': var closeIdx = querySlice.IndexOf('}'); if (closeIdx < 0) { position += querySlice.Length; querySlice = ReadOnlySpan.Empty; break; } var match = identifierRegex.Match(query, position, closeIdx); if (match.Success) { if (match.Groups[1].Success) parameterPositions.Add((position - 1, closeIdx + 2, match.Groups[1].Index - position + 1)); else parameterPositions.Add((position - 1, closeIdx + 2, -1)); } position += closeIdx + 1; querySlice = querySlice.Slice(closeIdx + 1); break; case '@': var simpleMatch = simpleIdentifierRegex.Match(query, position, querySlice.Length); if (simpleMatch.Success) { var len = simpleMatch.Groups[0].Length; parameterPositions.Add((position - 1, len + 1, -1)); position += len; querySlice = querySlice.Slice(len); } break; default: throw new NotSupportedException($"Internal error. Unexpected character \"{ch}\"."); } } return parameterPositions; } /// public override ValueTask DisposeAsync() { return base.DisposeAsync(); } /// protected override void Dispose(bool disposing) { base.Dispose(disposing); } internal sealed class SessionResources : IClickHouseSessionExternalResources { private readonly ClickHouseConnection? _connection; private readonly CancellationTokenSource? _tokenSource; public SessionResources(ClickHouseConnection? connection, CancellationTokenSource? tokenSource) { _connection = connection; _tokenSource = tokenSource; } public ValueTask Release(bool async) { _tokenSource?.Dispose(); return _connection?.Close(async) ?? default; } public async ValueTask ReleaseOnFailure(Exception? exception, bool async) { await Release(async); return null; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseConnection.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Data; using System.Data.Common; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Security; using System.Net.Sockets; using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using System.Threading; using System.Threading.Tasks; using System.Transactions; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Represents a connection to a ClickHouse database. This class cannot be inherited. /// public sealed class ClickHouseConnection : DbConnection { private const int MinBufferSize = 32; private readonly IClickHouseTypeInfoProvider? _typeInfoProvider; private ClickHouseConnectionState _connectionState; /// /// Gets or sets the string used to open a connection to a ClickHouse database server. /// [AllowNull] public override string ConnectionString { get { var state = _connectionState; if (state.Settings == null) return string.Empty; return new ClickHouseConnectionStringBuilder(state.Settings).ConnectionString; } set { var newSettings = value == null ? null : new ClickHouseConnectionStringBuilder(value).BuildSettings(); var state = _connectionState; while (true) { if (ReferenceEquals(state.Settings, newSettings)) break; if (state.State != ConnectionState.Closed && state.State != ConnectionState.Broken) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection string can not be modified because the connection is active."); var newState = new ClickHouseConnectionState(state.State, state.TcpClient, newSettings, unchecked(state.Counter + 1)); if (TryChangeConnectionState(state, newState, out state)) break; } } } /// /// The connection doesn't support a timeout. This property always returns . /// /// . public override int ConnectionTimeout => Timeout.Infinite; /// /// Gets the name of the database specified in the connection settings. /// /// The name of the database specified in the connection settings. The default value is an empty string. public override string Database => _connectionState.Settings?.Database ?? string.Empty; /// /// Gets the name of the database server specified in the connection settings. /// The name of the server contains the hostname and the port. If the port is equal to the default ClickHouse server port (9000) the /// name of the server will conatin only hostname. /// /// The name of the database server specified in the connection settings. The default value is an empty string. public override string DataSource { get { var state = _connectionState; if (state.Settings == null) return string.Empty; return state.Settings.Host + (state.Settings.Port != ClickHouseConnectionStringBuilder.DefaultPort ? ":" + state.Settings.Port : string.Empty); } } /// /// When the connection is open gets the version of the ClickHouse database server. /// /// The version of the ClickHouse database server. The default value is an empty string. public override string ServerVersion => _connectionState.TcpClient?.ServerInfo.Version.ToString() ?? string.Empty; /// /// Gets the state of the connection. /// /// The state of the connection. public override ConnectionState State => _connectionState.State; /// /// Gets or sets the callback for custom validation of the server's certificate. When the callback is set /// other TLS certificate validation options are ignored. /// public RemoteCertificateValidationCallback? RemoteCertificateValidationCallback { get; set; } internal TimeSpan? CommandTimeSpan { get { var commandTimeout = _connectionState.Settings?.CommandTimeout; if (commandTimeout == null) return null; return TimeSpan.FromSeconds(commandTimeout.Value); } } /// /// Gets the default mode of passing parameters to the query for the connection. /// /// The default mode of passing parameters to the query for the connection. The default value is . public ClickHouseParameterMode ParametersMode { get { var mode = _connectionState.Settings?.ParametersMode; if (mode == null || mode.Value == ClickHouseParameterMode.Inherit) return ClickHouseParameterMode.Default; return mode.Value; } } /// /// When the connection is open gets the server info. /// public ClickHouseServerInfo? ServerInfo => _connectionState.TcpClient?.ServerInfo; /// /// Initializes a new instance of class. /// public ClickHouseConnection() { _connectionState = new ClickHouseConnectionState(); } /// /// Initializes a new instance of with the settings and the default type provider. /// /// The connection string. public ClickHouseConnection(string connectionString) : this(connectionString, null) { } /// /// Initializes a new instance of with the settings. /// /// The connection string. /// Optional parameter. The provider of types for the connection. If the value is not specified the default type provider () will be used. public ClickHouseConnection(string connectionString, IClickHouseTypeInfoProvider? typeInfoProvider) : this(new ClickHouseConnectionStringBuilder(connectionString), typeInfoProvider) { } /// /// Initializes a new instance of with the settings and the default type provider. /// /// The connection string builder which will be used for building the connection settings. public ClickHouseConnection(ClickHouseConnectionStringBuilder stringBuilder) : this(stringBuilder, null) { } /// /// Initializes a new instance of with the settings. /// /// The connection string builder which will be used for building the connection settings. /// Optional parameter. The provider of types for the connection. If the value is not specified the default type provider () will be used. public ClickHouseConnection(ClickHouseConnectionStringBuilder stringBuilder, IClickHouseTypeInfoProvider? typeInfoProvider) { if (stringBuilder == null) throw new ArgumentNullException(nameof(stringBuilder)); var connectionSettings = stringBuilder.BuildSettings(); _connectionState = new ClickHouseConnectionState(ConnectionState.Closed, null, connectionSettings, 0); _typeInfoProvider = typeInfoProvider; } /// /// Initializes a new instance of with the settings and the default type provider. /// /// The connection settings. public ClickHouseConnection(ClickHouseConnectionSettings connectionSettings) : this(connectionSettings, null) { } /// /// Initializes a new instance of with the settings. /// /// The connection settings. /// Optional parameter. The provider of types for the connection. If the value is not specified the default type provider () will be used. public ClickHouseConnection(ClickHouseConnectionSettings connectionSettings, IClickHouseTypeInfoProvider? typeInfoProvider) { if (connectionSettings == null) throw new ArgumentNullException(nameof(connectionSettings)); _connectionState = new ClickHouseConnectionState(ConnectionState.Closed, null, connectionSettings, 0); _typeInfoProvider = typeInfoProvider; } /// /// Not supported. The database cannot be changed while the connection is open. /// /// Always throws . public override void ChangeDatabase(string databaseName) { throw new NotSupportedException(); } /// public override Task ChangeDatabaseAsync(string databaseName, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } /// /// Closes the connection to the database. /// public override void Close() { TaskHelper.WaitNonAsyncTask(Close(false)); } /// public override async Task CloseAsync() { await Close(true); } /// /// Not supported. Transactions are not supported by the ClickHouse server. /// /// Always throws . public override void EnlistTransaction(Transaction? transaction) { throw new NotSupportedException(); } /// /// Not supported. Schema information is not implemented. /// /// Always throws . public override DataTable GetSchema() { throw new NotImplementedException(); } /// public override DataTable GetSchema(string collectionName) { throw new NotImplementedException(); } /// public override DataTable GetSchema(string collectionName, string?[] restrictionValues) { throw new NotImplementedException(); } /// /// Opens a database connection. /// public override void Open() { TaskHelper.WaitNonAsyncTask(Open(false, CancellationToken.None)); } /// /// Opens a database connection asyncronously. /// /// The cancellation instruction. /// A representing asyncronous operation. public override async Task OpenAsync(CancellationToken cancellationToken) { await Open(true, cancellationToken); } /// /// Not supported. Transactions are not supported by the ClickHouse server. /// /// Always throws . protected override DbTransaction BeginDbTransaction(System.Data.IsolationLevel isolationLevel) { throw new NotSupportedException(); } /// /// Not supported. Transactions are not supported by the ClickHouse server. /// /// Always throws . protected override ValueTask BeginDbTransactionAsync(System.Data.IsolationLevel isolationLevel, CancellationToken cancellationToken) { throw new NotSupportedException(); } /// /// Creates and returns a object associated with the connection. /// /// A object. public new ClickHouseCommand CreateCommand() { return new ClickHouseCommand(this); } /// /// Creates and returns a object associated with the connection. /// /// The text for a new command. /// A object. public ClickHouseCommand CreateCommand(string commandText) { return new ClickHouseCommand(this) {CommandText = commandText}; } /// protected override DbCommand CreateDbCommand() { return CreateCommand(); } /// /// Creates and returns a object. /// /// The INSERT statement. /// A object. /// /// The command () must be a valid INSERT statement ending with VALUES. For example, /// INSERT INTO table(field1, ... fieldN) VALUES /// public ClickHouseColumnWriter CreateColumnWriter(string insertFormatCommand) { return TaskHelper.WaitNonAsyncTask(CreateColumnWriter(insertFormatCommand, false, CancellationToken.None)); } /// /// Asyncronously creates and returns a object. /// /// The INSERT statement. /// The cancellation instruction. /// A representing asyncronous operation. /// /// The command () must be a valid INSERT statement ending with VALUES. For example, /// INSERT INTO table(field1, ... fieldN) VALUES /// public async Task CreateColumnWriterAsync(string insertFormatCommand, CancellationToken cancellationToken) { return await CreateColumnWriter(insertFormatCommand, true, cancellationToken); } private async ValueTask CreateColumnWriter(string insertFormatCommand, bool async, CancellationToken cancellationToken) { var connectionState = _connectionState; if (connectionState.TcpClient == null) { Debug.Assert(connectionState.State != ConnectionState.Open); throw new ClickHouseException(ClickHouseErrorCodes.ConnectionClosed, "The connection is closed."); } ClickHouseTcpClient.Session? session = null; bool cancelOnFailure = false; try { session = await connectionState.TcpClient.OpenSession(async, null, CancellationToken.None, cancellationToken); var messageBuilder = new ClientQueryMessage.Builder {QueryKind = QueryKind.InitialQuery, Query = insertFormatCommand}; var query = await session.SendQuery(messageBuilder, null, async, cancellationToken); cancelOnFailure = true; var data = await ClickHouseColumnWriter.ReadTableMetadata(session, query.Query, async, cancellationToken); return new ClickHouseColumnWriter(session, query, data.Header.Columns); } catch (ClickHouseServerException) { if (session != null) await session.Dispose(async); throw; } catch(ClickHouseHandledException) { if (session != null) await session.Dispose(async); throw; } catch(Exception ex) { if (session != null) { var aggrEx = await session.SetFailed(ex, cancelOnFailure, async); if (aggrEx != null) throw aggrEx; } throw; } } /// /// For an open connection gets the default timezone of the ClickHouse server. /// /// The default timezone of the ClickHouse server. /// Throws if the connection is not open. public TimeZoneInfo GetServerTimeZone() { var connectionState = _connectionState; var serverInfo = connectionState.TcpClient?.ServerInfo; if (serverInfo == null || connectionState.State != ConnectionState.Open) throw new ClickHouseException(ClickHouseErrorCodes.ConnectionClosed, "The connection is closed."); return TimeZoneHelper.GetTimeZoneInfo(serverInfo.Timezone); } /// /// Closes the connection and releases resources associated with it. /// protected override void Dispose(bool disposing) { var connectionState = _connectionState; if (connectionState == null) { // This is possible when GC calls finalizer for System.ComponentModel.Component, but the connection // object was not created properly (an error occured in the constructor). return; } var counter = connectionState.Counter; while (connectionState.Counter == counter) { var targetState = connectionState.State == ConnectionState.Closed ? ConnectionState.Closed : ConnectionState.Broken; if (connectionState.State == targetState && connectionState.TcpClient == null) break; var tcpClient = connectionState.TcpClient; if (!TryChangeConnectionState(connectionState, targetState, null, out connectionState, out _)) continue; tcpClient?.Dispose(); break; } base.Dispose(disposing); } private async ValueTask Open(bool async, CancellationToken cancellationToken) { if (!BitConverter.IsLittleEndian) { throw new NotSupportedException( "An architecture of the processor is not supported. Only little-endian architectures are supported." + Environment.NewLine + "Please, report an issue if you see this message (https://github.com/Octonica/ClickHouseClient/issues)."); } var connectionState = _connectionState; switch (connectionState.State) { case ConnectionState.Closed: break; case ConnectionState.Open: return; // Re-entrance is allowed case ConnectionState.Connecting: throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection is already opening."); case ConnectionState.Broken: throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection is broken."); default: throw new NotSupportedException($"Internal error. The state {_connectionState} is not supported."); } if (!TryChangeConnectionState(connectionState, ConnectionState.Connecting, out connectionState, out var onStateChanged)) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The state of the connection was modified."); var stateChangeEx = onStateChanged(this); var connectionSettings = connectionState.Settings; if (stateChangeEx != null || connectionSettings == null) { var initialEx = stateChangeEx ?? new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection is not initialized."); if (!TryChangeConnectionState(connectionState, ConnectionState.Closed, out _, out onStateChanged)) throw new AggregateException(initialEx, new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The state of the connection was modified.")); var stateChangeEx2 = onStateChanged(this); if (stateChangeEx2 != null) throw new AggregateException(initialEx, stateChangeEx2); if (stateChangeEx != null) throw new ClickHouseException(ClickHouseErrorCodes.CallbackError, "External callback error. See the inner exception for details.", stateChangeEx); throw initialEx; } const int defaultHttpPort = 8123; TcpClient? client = null; SslStream? sslStream = null; ClickHouseBinaryProtocolWriter? writer = null; ClickHouseBinaryProtocolReader? reader = null; try { try { client = new TcpClient {SendTimeout = connectionSettings.ReadWriteTimeout, ReceiveTimeout = connectionSettings.ReadWriteTimeout}; if (async) await client.ConnectAsync(connectionSettings.Host, connectionSettings.Port); else client.Connect(connectionSettings.Host, connectionSettings.Port); } catch { client?.Client?.Close(0); client?.Dispose(); client = null; throw; } if (connectionSettings.TlsMode == ClickHouseTlsMode.Require) { var certValidationCallback = RemoteCertificateValidationCallback; if (certValidationCallback == null && (connectionSettings.RootCertificate != null || !connectionSettings.ServerCertificateHash.IsEmpty)) certValidationCallback = (_, cert, chain, errors) => ValidateServerCertificate(connectionSettings, cert, chain, errors); sslStream = new SslStream(client.GetStream(), true, certValidationCallback); try { if (async) await sslStream.AuthenticateAsClientAsync(new SslClientAuthenticationOptions { TargetHost = connectionSettings.Host }, cancellationToken); else sslStream.AuthenticateAsClient(connectionSettings.Host); } catch(AuthenticationException authEx) { throw new ClickHouseException(ClickHouseErrorCodes.TlsError, $"TLS handshake error.", authEx); } } var stream = sslStream ?? (Stream)client.GetStream(); writer = new ClickHouseBinaryProtocolWriter(stream, Math.Max(connectionSettings.BufferSize, MinBufferSize)); var clientHello = new ClientHelloMessage.Builder { ClientName = connectionSettings.ClientName, ClientVersion = connectionSettings.ClientVersion, User = connectionSettings.User, Database = connectionSettings.Database, Password = connectionSettings.Password, ProtocolRevision = ClickHouseProtocolRevisions.CurrentRevision, QuotaKey = connectionSettings.QuotaKey }.Build(); clientHello.Write(writer); await writer.Flush(async, cancellationToken); reader = new ClickHouseBinaryProtocolReader(stream, Math.Max(connectionSettings.BufferSize, MinBufferSize)); var message = await reader.ReadMessage(clientHello.ProtocolRevision, false, async, cancellationToken); switch (message.MessageCode) { case ServerMessageCode.Hello: var helloMessage = (ServerHelloMessage) message; var serverInfo = helloMessage.ServerInfo; if (serverInfo.Revision >= ClickHouseProtocolRevisions.MinRevisionWithAddendum) { // Despite receiving the server hello message, we don't know yet if there was an error on the server side. // The server could reply with an error after receiveng an addendum. If there is no error, the server will not reply at all. // We can't rely on checking the state of the network channel, because the absense of bytes doesn't guarantee the success, // an error message could be delayed. Instead, we are going to add the ping message after the addendum, forcing the server // to send a reply in any case. clientHello.WriteAddendum(writer); writer.Write7BitInt32((int)ClientMessageCode.Ping); await writer.Flush(async, cancellationToken); var extraMessage = await reader.ReadMessage(Math.Min(clientHello.ProtocolRevision, serverInfo.Revision), false, async, cancellationToken); if (extraMessage.MessageCode != ServerMessageCode.Pong) { if (extraMessage.MessageCode == ServerMessageCode.Error) throw ((ServerErrorMessage)extraMessage).Exception; throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Unexpected message code (0x{extraMessage.MessageCode:X}) received from the server."); } } bool hasExtraByte = reader.TryPeekByte(out var extraByte); if (!hasExtraByte && client.Available > 0) { hasExtraByte = true; extraByte = await reader.ReadByte(async, cancellationToken); } if (hasExtraByte) { throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Expected the end of the data. Unexpected byte (0x{extraByte:X}) received from the server."); } var configuredTypeInfoProvider = (_typeInfoProvider ?? ClickHouseTypeInfoProvider.Instance).Configure(serverInfo); var tcpClient = new ClickHouseTcpClient(client, reader, writer, connectionSettings, serverInfo, configuredTypeInfoProvider, sslStream); if (!TryChangeConnectionState(connectionState, ConnectionState.Open, tcpClient, out _, out onStateChanged)) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The state of the connection was modified."); break; case ServerMessageCode.Error: throw ((ServerErrorMessage) message).Exception; default: if ((int) message.MessageCode == 'H') { // It looks like HTTP string httpDetectedMessage; if (connectionSettings.Port == defaultHttpPort) { // It's definitely HTTP httpDetectedMessage = $"Detected an attempt to connect by HTTP protocol with the default port {defaultHttpPort}. "; } else { httpDetectedMessage = $"Internal error. Unexpected message code (0x{message.MessageCode:X}) received from the server. " + "This error may by caused by an attempt to connect with HTTP protocol. "; } httpDetectedMessage += $"{ClickHouseConnectionStringBuilder.DefaultClientName} supports only ClickHouse native protocol. " + $"The default port for the native protocol is {ClickHouseConnectionStringBuilder.DefaultPort}."; throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, httpDetectedMessage); } if ((int) message.MessageCode == 0x15) { // 0x15 stands for TLS alert message var sslAlertMessage = $"Unexpected message code (0x{message.MessageCode:X}) received from the server. " + "This code may indicate that the server requires establishing a connection over TLS."; throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, sslAlertMessage); } throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Unexpected message code (0x{message.MessageCode:X}) received from the server."); } } catch (Exception ex) { reader?.Dispose(); writer?.Dispose(); sslStream?.Dispose(); client?.Client?.Close(0); client?.Dispose(); if (TryChangeConnectionState(connectionState, ConnectionState.Closed, out _, out onStateChanged)) stateChangeEx = onStateChanged(this); if (connectionSettings.Port == defaultHttpPort && ex is IOException) { var extraMessage = $"{ex.Message} This error may be caused by an attempt to connect to the default HTTP port ({defaultHttpPort}). " + $"{ClickHouseConnectionStringBuilder.DefaultClientName} supports only ClickHouse native protocol. " + $"The default port for the native protocol is {ClickHouseConnectionStringBuilder.DefaultPort}."; var extraEx = new IOException(extraMessage, ex); if (stateChangeEx != null) throw new AggregateException(extraEx, stateChangeEx); throw extraEx; } if (stateChangeEx != null) throw new AggregateException(ex, stateChangeEx); throw; } stateChangeEx = onStateChanged.Invoke(this); if (stateChangeEx != null) throw new ClickHouseException(ClickHouseErrorCodes.CallbackError, "External callback error. See the inner exception for details.", stateChangeEx); } /// /// Send ping to the server and wait for response. /// /// /// Returns true if ping was successful. /// Returns false if the connection is busy with a command execution. /// public bool TryPing() { return TaskHelper.WaitNonAsyncTask(TryPing(false, CancellationToken.None)); } /// public Task TryPingAsync() { return TryPingAsync(CancellationToken.None); } /// public async Task TryPingAsync(CancellationToken cancellationToken) { return await TryPing(true, cancellationToken); } private async ValueTask TryPing(bool async, CancellationToken cancellationToken) { if (_connectionState.TcpClient?.State == ClickHouseTcpClientState.Active) return false; ClickHouseTcpClient.Session? session = null; try { using (var ts = new CancellationTokenSource(TimeSpan.FromMilliseconds(5))) { try { session = await OpenSession(async, null, CancellationToken.None, ts.Token); } catch (OperationCanceledException ex) { if (ex.CancellationToken == ts.Token) return false; throw; } } await session.SendPing(async, cancellationToken); var responseMsg = await session.ReadMessage(async, cancellationToken); switch (responseMsg.MessageCode) { case ServerMessageCode.Pong: return true; case ServerMessageCode.Error: // Something else caused this error. Keep it in InnerException for debug. throw new ClickHouseException( ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Unexpected message code (0x{responseMsg.MessageCode:X}) received from the server as a response to ping.", ((ServerErrorMessage)responseMsg).Exception); default: throw new ClickHouseException( ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Unexpected message code (0x{responseMsg.MessageCode:X}) received from the server as a response to ping."); } } catch (ClickHouseHandledException) { throw; } catch (Exception ex) { if (session != null) { await session.SetFailed(ex, false, async); session = null; } throw; } finally { if (session != null) { if (async) await session.DisposeAsync(); else session.Dispose(); } } } internal ValueTask OpenSession(bool async, IClickHouseSessionExternalResources? externalResources, CancellationToken sessionCancellationToken, CancellationToken cancellationToken) { var connectionSession = new ConnectionSession(this, externalResources); return connectionSession.OpenSession(async, sessionCancellationToken, cancellationToken); } internal async ValueTask Close(bool async) { var connectionState = _connectionState; var counter = connectionState.Counter; while (connectionState.Counter == counter) { var tcpClient = connectionState.TcpClient; Func? onStateChanged; Exception? stateChangedEx; switch (connectionState.State) { case ConnectionState.Closed: return; // Re-entrance is allowed case ConnectionState.Open: ClickHouseTcpClient.Session? session = null; try { // Acquire session for preventing access to the communication object var sessionTask = tcpClient?.OpenSession(async, null, CancellationToken.None, CancellationToken.None); if (sessionTask != null) session = await sessionTask.Value; } catch (ObjectDisposedException) { if (!TryChangeConnectionState(connectionState, ConnectionState.Closed, null, out connectionState, out onStateChanged)) continue; stateChangedEx = onStateChanged(this); if (stateChangedEx != null) throw new ClickHouseException(ClickHouseErrorCodes.CallbackError, "External callback error. See the inner exception for details.", stateChangedEx); return; } catch { if (session != null) await session.Dispose(async); throw; } if (!TryChangeConnectionState(connectionState, ConnectionState.Closed, null, out connectionState, out onStateChanged)) { if (session != null) await session.Dispose(async); continue; } tcpClient?.Dispose(); stateChangedEx = onStateChanged(this); if (stateChangedEx != null) throw new ClickHouseException(ClickHouseErrorCodes.CallbackError, "External callback error. See the inner exception for details.", stateChangedEx); return; case ConnectionState.Broken: if (!TryChangeConnectionState(connectionState, ConnectionState.Closed, null, out connectionState, out onStateChanged)) continue; tcpClient?.Dispose(); stateChangedEx = onStateChanged(this); if (stateChangedEx != null) throw new ClickHouseException(ClickHouseErrorCodes.CallbackError, "External callback error. See the inner exception for details.", stateChangedEx); break; case ConnectionState.Connecting: throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection is opening. It can't be closed."); default: throw new NotSupportedException($"Internal error. The state {connectionState.State} is not supported."); } } } private bool TryChangeConnectionState(ClickHouseConnectionState from, ClickHouseConnectionState to, out ClickHouseConnectionState actualState) { actualState = Interlocked.CompareExchange(ref _connectionState, to, from); if (ReferenceEquals(actualState, from)) { actualState = to; return true; } return false; } private bool TryChangeConnectionState( ClickHouseConnectionState state, ConnectionState newState, ClickHouseTcpClient? client, out ClickHouseConnectionState actualState, [NotNullWhen(true)] out Func? onStateChanged) { var counter = state.State != ConnectionState.Connecting && newState == ConnectionState.Connecting ? unchecked(state.Counter + 1) : state.Counter; var nextState = new ClickHouseConnectionState(newState, client, state.Settings, counter); if (TryChangeConnectionState(state, nextState, out actualState)) { onStateChanged = CreateConnectionStateChangedCallback(state.State, actualState.State); return true; } onStateChanged = null; return false; } private bool TryChangeConnectionState( ClickHouseConnectionState state, ConnectionState newState, out ClickHouseConnectionState actualState, [NotNullWhen(true)] out Func? onStateChanged) { return TryChangeConnectionState(state, newState, state.TcpClient, out actualState, out onStateChanged); } private static Func CreateConnectionStateChangedCallback(ConnectionState originalState, ConnectionState currentState) { if (originalState == currentState) return _ => null; return FireEvent; Exception? FireEvent(ClickHouseConnection connection) { try { connection.OnStateChange(new StateChangeEventArgs(originalState, currentState)); } catch (Exception ex) { return ex; } return null; } } private static bool ValidateServerCertificate(ClickHouseConnectionSettings connectionSettings, X509Certificate? cert, X509Chain? chain, SslPolicyErrors errors) { if (errors == SslPolicyErrors.None) return true; if (cert == null) return false; if (!connectionSettings.ServerCertificateHash.IsEmpty) { var certHash = cert.GetCertHash(); if (connectionSettings.ServerCertificateHash.Span.SequenceEqual(certHash)) return true; } if (chain != null && connectionSettings.RootCertificate != null) { if ((errors & ~SslPolicyErrors.RemoteCertificateChainErrors) != SslPolicyErrors.None) return false; var collection = CertificateHelper.LoadFromFile(connectionSettings.RootCertificate); #if NET5_0_OR_GREATER chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; chain.ChainPolicy.CustomTrustStore.AddRange(collection); var isValid = chain.Build(cert as X509Certificate2 ?? new X509Certificate2(cert)); return isValid; #else foreach (var chainElement in chain.ChainElements) { if (chainElement.ChainElementStatus.Length != 0) { bool ignoreError = true; foreach (var status in chainElement.ChainElementStatus) { if (status.Status == X509ChainStatusFlags.UntrustedRoot) continue; ignoreError = false; break; } if (!ignoreError) break; } if (collection.Contains(chainElement.Certificate)) return true; } #endif } return false; } private class ConnectionSession : IClickHouseSessionExternalResources { private readonly ClickHouseConnection _connection; private readonly ClickHouseConnectionState _state; private readonly IClickHouseSessionExternalResources? _externalResources; public ConnectionSession(ClickHouseConnection connection, IClickHouseSessionExternalResources? externalResources) { _connection = connection; _state = _connection._connectionState; _externalResources = externalResources; var tcpClient = _state.TcpClient; if (tcpClient == null) { Debug.Assert(_state.State != ConnectionState.Open); throw new ClickHouseException(ClickHouseErrorCodes.ConnectionClosed, "The connection is closed."); } if (_state.State != ConnectionState.Open) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "The connection is closed."); } public ValueTask OpenSession(bool async, CancellationToken sessionCancellationToken, CancellationToken cancellationToken) { Debug.Assert(_state.TcpClient != null); return _state.TcpClient.OpenSession(async, this, sessionCancellationToken, cancellationToken); } public ValueTask Release(bool async) { return _externalResources?.Release(async) ?? default; } public async ValueTask ReleaseOnFailure(Exception? exception, bool async) { Exception? ex = null; if (_connection.TryChangeConnectionState(_state, ConnectionState.Broken, null, out _, out var onStateChanged)) ex = onStateChanged(_connection); Exception? externalEx = null; if (_externalResources != null) externalEx = await _externalResources.ReleaseOnFailure(exception, async); if (ex != null && externalEx != null) return new AggregateException(ex, externalEx); return externalEx ?? ex; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseConnectionSettings.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient { /// /// Represents an immutable set of properties applied to the connection. This class can't be inherited. /// public sealed class ClickHouseConnectionSettings { /// /// Gets the name or the IP address of the host. /// public string Host { get; } /// /// Gets the IP port of the server. /// public ushort Port { get; } /// /// Gets the name of the user. /// public string User { get; } /// /// Gets the password. /// public string? Password { get; } /// /// Get the name of the default database. /// public string? Database { get; } /// /// Gets the network socket timeout in milliseconds. This timeout will be used to initialize properties and /// . /// public int ReadWriteTimeout { get; } /// /// Gets the name of the client. The name of the client is passed to the ClickHouse server as a part of the client's identifier. /// public string ClientName { get; } /// /// Gets the version of the client. The first two parts of the version ( and ) /// are passed to the ClickHouse server as a part of the client's identifier. /// public ClickHouseVersion ClientVersion { get; } /// /// Gets the preferred size of the internal buffer in bytes. /// public int BufferSize { get; } /// /// Gets the value indicating whether the compression (LZ4) of data is enabled. /// public bool Compress { get; } /// /// Gets the command timeout in seconds. This timeout will be used to initialize the property of the command. /// public int CommandTimeout { get; } /// /// Gets the TLS mode for the connection. See for details. /// public ClickHouseTlsMode TlsMode { get; } /// /// Gets the path to the file that contains a certificate (*.crt) or a list of certificates (*.pem). /// When performing TLS hanshake any of these certificates will be treated as a valid root for the certificate chain. /// public string? RootCertificate { get; } /// /// Gets the hash of the server's certificate. When performing TLS handshake the remote certificate with the specified /// hash will be treated as a valid certificate despite any other certificate chain validation errors (e.g. invalid hostname). /// public ReadOnlyMemory ServerCertificateHash { get; } /// /// Gets the default mode of passing parameters to the query for the connection. /// public ClickHouseParameterMode ParametersMode { get; } /// /// Gets the 'quota key' passed with the query. This key is used by the ClickHouse server for tracking quotas. /// public string? QuotaKey { get; } internal readonly int CompressionBlockSize = 1024 * 8; // Maybe it should be configurable internal ClickHouseConnectionSettings(ClickHouseConnectionStringBuilder builder) { if (string.IsNullOrWhiteSpace(builder.Host)) throw new ArgumentException("The host is not defined.", nameof(builder)); if (builder.BufferSize <= 0) throw new ArgumentException("The size of the buffer must be a positive number.", nameof(builder)); Host = builder.Host; Port = builder.Port; User = builder.User; Password = string.IsNullOrEmpty(builder.Password) ? null : builder.Password; Database = string.IsNullOrEmpty(builder.Database) ? null : builder.Database; ReadWriteTimeout = builder.ReadWriteTimeout; BufferSize = builder.BufferSize; ClientName = builder.ClientName; ClientVersion = builder.ClientVersion; Compress = builder.Compress; CommandTimeout = builder.CommandTimeout; TlsMode = builder.TlsMode; RootCertificate = builder.RootCertificate; ServerCertificateHash = ParseHashString(builder.ServerCertificateHash); ParametersMode = builder.ParametersMode; QuotaKey = string.IsNullOrEmpty(builder.QuotaKey) ? null : builder.QuotaKey; } private static byte[]? ParseHashString(string? hashString) { if (string.IsNullOrEmpty(hashString)) return null; int resultPos = 0; var result = new byte[hashString.Length / 2]; for (int i = 0; i < hashString.Length; i++) { var ch = hashString[i]; if (char.IsWhiteSpace(ch) || ch == '-') continue; if (i + 1 == hashString.Length) throw new ArgumentException("Unexpected end of the hash string. Expected at least one more significant character.", nameof(hashString)); byte byteVal; if (ch >= '0' && ch <= '9') byteVal = (byte)(ch - '0'); else if (ch >= 'a' && ch <= 'f') byteVal = (byte)(ch - 'a' + 0xA); else if (ch >= 'A' && ch <= 'F') byteVal = (byte)(ch - 'A' + 0xA); else throw new ArgumentException($"Unexpected character '{ch}' at the position {i} in the hash string.", nameof(hashString)); byteVal <<= 4; ch = hashString[++i]; if (ch >= '0' && ch <= '9') byteVal |= (byte)(ch - '0'); else if (ch >= 'a' && ch <= 'f') byteVal |= (byte)(ch - 'a' + 0xA); else if (ch >= 'A' && ch <= 'F') byteVal |= (byte)(ch - 'A' + 0xA); else throw new ArgumentException($"Unexpected character '{ch}' at the position {i} in the hash string.", nameof(hashString)); result[resultPos++] = byteVal; } if (resultPos == 0) return null; if (result.Length != resultPos) Array.Resize(ref result, resultPos); return result; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseConnectionState.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Data; namespace Octonica.ClickHouseClient { internal sealed class ClickHouseConnectionState { public ConnectionState State { get; } public ClickHouseConnectionSettings? Settings { get; } public int Counter { get; } public ClickHouseTcpClient? TcpClient { get; } public ClickHouseConnectionState() : this(ConnectionState.Closed, null, null, 0) { } public ClickHouseConnectionState(ConnectionState state, ClickHouseTcpClient? tcpClient, ClickHouseConnectionSettings? settings, int counter) { State = state; TcpClient = tcpClient; Settings = settings; Counter = counter; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseConnectionStringBuilder.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Data.Common; using System.Diagnostics.CodeAnalysis; using System.Globalization; namespace Octonica.ClickHouseClient { /// /// Provides a set of methods for working with connection settings and connection strings. /// public class ClickHouseConnectionStringBuilder : DbConnectionStringBuilder { private static readonly HashSet AllProperties; /// /// The default IP port of the server (9000). /// public const ushort DefaultPort = 9000; /// /// The default name of the user ('default'). /// public const string DefaultUser = "default"; /// /// The default network socket timeout in milliseconds (10000). /// public const int DefaultReadWriteTimeout = 10000; /// /// The default command timeout in seconds (15). /// public const int DefaultCommandTimeout = 15; /// /// The default size of the internal buffer in bytes (4096). /// public const int DefaultBufferSize = 4096; /// /// The default name of the client ('Octonica.ClickHouseClient'). /// public const string DefaultClientName = "Octonica.ClickHouseClient"; /// /// The default value (). indicating whether the compression (LZ4) is enabled. /// public const bool DefaultCompress = true; /// /// The default version of the client. This value is equal to the version of the assembly Octonica.ClickHouseClient. /// public static readonly ClickHouseVersion DefaultClientVersion; /// /// The default value for the TLS mode is . /// public const ClickHouseTlsMode DefaultTlsMode = ClickHouseTlsMode.Disable; /// /// The default value for the mode of passing parameters to the query is . /// public const ClickHouseParameterMode DefaultParametersMode = ClickHouseParameterMode.Default; /// /// Gets or sets the name or the IP address of the host. /// /// The name or the IP address of the host. public string? Host { get => GetString(nameof(Host)); set => this[nameof(Host)] = value; } /// /// Gets or sets the IP port of the server. /// /// The IP port of the server. The default value is . public ushort Port { get => (ushort) GetInt32OrDefault(nameof(Port), DefaultPort); set => this[nameof(Port)] = value; } /// /// Gets or sets the name of the user. /// /// The name of the user. The default value is . public string User { get => GetStringOrDefault(nameof(User), DefaultUser); set => this[nameof(User)] = value; } /// /// Gets or sets the password. /// /// The password. public string? Password { get => GetString(nameof(Password)); set => this[nameof(Password)] = value; } /// /// Gets or sets the name of the default database. /// /// The name of the default database. if the default database in not specified. public string? Database { get => GetString(nameof(Database)); set => this[nameof(Database)] = value; } /// /// Gets or sets the network socket timeout in milliseconds. This timeout will be used to initialize properties and /// . /// /// The network socket timeout in milliseconds. The default value is . public int ReadWriteTimeout { get => GetInt32OrDefault(nameof(ReadWriteTimeout), DefaultReadWriteTimeout); set => this[nameof(ReadWriteTimeout)] = value; } /// /// Gets or sets the preferred size of the internal buffer in bytes. /// /// the preferred size of the internal buffer in bytes. The default value is . public int BufferSize { get => GetInt32OrDefault(nameof(BufferSize), DefaultBufferSize); set => this[nameof(BufferSize)] = value; } /// /// Gets or sets the name of the client. The name of the client is passed to the ClickHouse server as a part of the client's identifier. /// /// The name of the client. The default value is . public string ClientName { get => GetStringOrDefault(nameof(ClientName), DefaultClientName); set => this[nameof(ClientName)] = value; } /// /// Gets or sets the value indicating whether the compression (LZ4) of data is enabled. /// /// if compression is enabled; otherwise . The default value is . public bool Compress { get => GetBoolOrDefault(nameof(Compress), DefaultCompress); set => this[nameof(Compress)] = value; } /// /// Gets or sets the command timeout in seconds. This timeout will be used to initialize the property of the command. /// /// The command timeout in seconds. The default value is . public int CommandTimeout { get => GetInt32OrDefault(nameof(CommandTimeout), DefaultCommandTimeout); set => this[nameof(CommandTimeout)] = value; } /// /// Gets or sets the version of the client. The first two parts of the version ( and ) /// are passed to the ClickHouse server as a part of the client's identifier. /// /// The version of the client. The default value is . public ClickHouseVersion ClientVersion { get { var value = GetString(nameof(ClientVersion)); if (value == null) return DefaultClientVersion; return ClickHouseVersion.Parse(value); } set => this[nameof(ClientVersion)] = value.ToString(); } /// /// Gets or sets the TLS mode for the connection. See for details. /// /// The TLS mode for the connection. The default value is . public ClickHouseTlsMode TlsMode { get => GetEnumOrDefault(nameof(TlsMode), DefaultTlsMode); set => this[nameof(TlsMode)] = value == DefaultTlsMode ? null : value.ToString("G"); } /// /// Gets or sets the path to the file that contains a certificate (*.crt) or a list of certificates (*.pem). /// When performing TLS hanshake any of these certificates will be treated as a valid root for the certificate chain. /// /// The path to the file that contains a certificate (*.crt) or a list of certificates (*.pem). The default value is . public string? RootCertificate { get { var value = GetString(nameof(RootCertificate)); if (string.IsNullOrWhiteSpace(value)) return null; return value; } set => this[nameof(RootCertificate)] = value; } /// /// Gets or sets the hash of the server's certificate in the hexadecimal format. /// When performing TLS handshake the remote certificate with the specified hash will be treated as a valid certificate /// despite any other certificate chain validation errors (e.g. invalid hostname). /// /// The hash of the server's certificate in the hexadecimal format. The default value is . public string? ServerCertificateHash { get { var value = GetString(nameof(ServerCertificateHash)); if (string.IsNullOrWhiteSpace(value)) return null; return value; } set => this[nameof(ServerCertificateHash)] = value; } /// /// Gets the default mode of passing parameters to the query for the connection. /// /// The default mode of passing parameters to the query for the connection. The default value is . public ClickHouseParameterMode ParametersMode { get => GetEnumOrDefault(nameof(ParametersMode), DefaultParametersMode); set => this[nameof(ParametersMode)] = value == DefaultParametersMode ? null : value.ToString("G"); } /// /// Gets the 'quota key' passed with the query. This key is used by the ClickHouse server for tracking quotas. /// /// The value of 'quota key' passed with the query. public string? QuotaKey { get => GetString(nameof(QuotaKey)); set => this[nameof(QuotaKey)] = value; } static ClickHouseConnectionStringBuilder() { var asm = typeof(ClickHouseConnectionStringBuilder).Assembly; var version = asm.GetName().Version; DefaultClientVersion = new ClickHouseVersion(version?.Major ?? 1, version?.Minor ?? 0, version?.Build ?? 0); AllProperties = new HashSet(StringComparer.OrdinalIgnoreCase) { nameof(BufferSize), nameof(ClientName), nameof(ClientVersion), nameof(CommandTimeout), nameof(Compress), nameof(Database), nameof(Host), nameof(Password), nameof(ReadWriteTimeout), nameof(Port), nameof(User), nameof(TlsMode), nameof(RootCertificate), nameof(ServerCertificateHash), nameof(ParametersMode), nameof(QuotaKey) }; } /// /// Initializes a new instance of with the default settings. /// public ClickHouseConnectionStringBuilder() { } /// /// Initializes a new instance of with the settings specified in the connection string. /// /// The connection string. public ClickHouseConnectionStringBuilder(string connectionString) { ConnectionString = connectionString; } /// /// Initializes a new instance of with the specified. /// /// The settings. public ClickHouseConnectionStringBuilder(ClickHouseConnectionSettings settings) { if (settings == null) throw new ArgumentNullException(nameof(settings)); Host = settings.Host; Port = settings.Port; User = settings.User; Password = settings.Password; Database = settings.Database; ReadWriteTimeout = settings.ReadWriteTimeout; BufferSize = settings.BufferSize; Compress = settings.Compress; CommandTimeout = settings.CommandTimeout; TlsMode = settings.TlsMode; RootCertificate = settings.RootCertificate; ServerCertificateHash = HashToString(settings.ServerCertificateHash); ParametersMode = settings.ParametersMode; QuotaKey = settings.QuotaKey; if (settings.ClientName != DefaultClientName) ClientName = settings.ClientName; if (settings.ClientVersion != DefaultClientVersion) ClientVersion = settings.ClientVersion; } /// [AllowNull] public override object this[string keyword] { get => base[keyword]; set { if (!AllProperties.Contains(keyword)) throw new ArgumentException($"\"{keyword}\" is not a valid connection parameter name.", nameof(keyword)); base[keyword] = value; } } /// /// Creates and returns a new instance of the . /// /// A new instance of the . public ClickHouseConnectionSettings BuildSettings() { return new ClickHouseConnectionSettings(this); } private string? GetString(string key) { return TryGetValue(key, out var value) ? (string) value : null; } private string GetStringOrDefault(string key, string defaultValue) { if (!TryGetValue(key, out var value)) return defaultValue; return (string) value ?? defaultValue; } private int GetInt32OrDefault(string key, int defaultValue) { if (!TryGetValue(key, out var value)) return defaultValue; if (value is string strValue) { if (!int.TryParse(strValue.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var result)) throw new InvalidOperationException($"The value of the property \"{key}\" must be an integer value."); return result; } return (int?) value ?? defaultValue; } private bool GetBoolOrDefault(string key, bool defaultValue) { if (!TryGetValue(key, out var value)) return defaultValue; if (value is string strValue) { switch (strValue.Trim().ToLowerInvariant()) { case "on": case "true": case "1": return true; case "off": case "false": case "0": return false; default: throw new InvalidOperationException($"The value of the property \"{key}\" is not a valid boolean value."); } } return (bool?) value ?? defaultValue; } private TEnum GetEnumOrDefault(string key, TEnum defaultValue) where TEnum : struct { if (!TryGetValue(key, out var value) || value == null) return defaultValue; if(value is string strValue) { if (string.IsNullOrWhiteSpace(strValue)) return defaultValue; // Enum.TryParse parses an integer value and casts it into enum without additional check. // Check that the value is not an integer before performing an actual enum parsing. if (int.TryParse(strValue.Trim(), out _) || !Enum.TryParse(strValue, true, out var result)) throw new InvalidOperationException($"The value \"{strValue}\" is not a valid value for the property \"{key}\"."); return result; } return (TEnum)value; } private static string? HashToString(ReadOnlyMemory hashBytes) { if (hashBytes.Length == 0) return null; return string.Create(hashBytes.Length * 2, hashBytes, HashToString); } static void HashToString(Span span, ReadOnlyMemory hashBytes) { var byteSpan = hashBytes.Span; for (int i = 0; i < hashBytes.Length; i++) { var val = byteSpan[i] >> 4; span[i * 2] = (char)(val + (val >= 0xA ? 'A' - 0xA : '0')); val = byteSpan[i] & 0x0F; span[i * 2 + 1] = (char)(val + (val >= 0xA ? 'A' - 0xA : '0')); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDataReader.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024, 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; using System.Numerics; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Provides a way of reading a forward-only stream of rows from a ClickHouse database. This class cannot be inherited. /// public sealed class ClickHouseDataReader : ClickHouseDataReaderBase { private readonly ClickHouseTcpClient.Session _session; private readonly ClickHouseDataReaderRowLimit _rowLimit; private readonly bool _ignoreProfileEvents; private ulong _recordsAffected; private int _rowIndex = -1; private IServerMessage? _nextResultMessage; private ClickHouseTable _currentTable; private IClickHouseTableColumn[] _reinterpretedColumnsCache; private ClickHouseReaderColumnSettings[]? _columnSettings; /// /// Gets the current state of the reader. /// public ClickHouseDataReaderState State { get; private set; } /// /// Gets the query execution progress reported by the server. /// public ClickHouseQueryExecutionProgress ExecutionProgress { get; private set; } /// /// Gets the number of affected rows. /// /// A non-negative number of rows affected by the query. /// The number of affected rows is greater than . /// Use the property if the query can affect more than rows . public override int RecordsAffected { get { if (_recordsAffected <= int.MaxValue) return (int) _recordsAffected; throw new OverflowException($"The number of affected records is too large. Use the property \"{nameof(RecordsAffectedLong)}\" to get this number."); } } /// /// Gets the number of affected rows. /// /// A number of rows affected by the query. public ulong RecordsAffectedLong => _recordsAffected; /// /// Gets a value that indicates whether the reader contains one or more rows. /// /// /// if the reader contains one or more rows; otherwise . public override bool HasRows => _rowIndex < 0 || _recordsAffected > 0; /// /// Gets the value indicating whether the reader is closed. /// /// if the reader is closed; otherwise . public override bool IsClosed => State == ClickHouseDataReaderState.Closed || State == ClickHouseDataReaderState.Broken; /// /// Gets the number of columns. /// public override int FieldCount => _currentTable.Header.Columns.Count; /// /// Gets a value that indicates the depth of nesting for the current row. /// /// Always returns 0. public override int Depth => 0; internal ClickHouseDataReader(ClickHouseTable table, ClickHouseTcpClient.Session session, ClickHouseQueryExecutionProgress executionProgress, ClickHouseDataReaderRowLimit rowLimit, bool ignoreProfileEvents) { _currentTable = table.Header == null || table.Columns == null ? throw new ArgumentNullException(nameof(table)) : table; _session = session ?? throw new ArgumentNullException(nameof(session)); _rowLimit = rowLimit; _ignoreProfileEvents = ignoreProfileEvents; _reinterpretedColumnsCache = new IClickHouseTableColumn[_currentTable.Columns.Count]; _recordsAffected = checked((ulong) _currentTable.Header.RowCount); ExecutionProgress = executionProgress; State = _rowLimit == ClickHouseDataReaderRowLimit.Zero ? ClickHouseDataReaderState.ClosePending : ClickHouseDataReaderState.Data; } /// /// An empty, closed reader. Only for a disposed session. /// internal ClickHouseDataReader(ClickHouseTcpClient.Session session) { _currentTable = new ClickHouseTable(new BlockHeader(null, new List(0).AsReadOnly(), 0), new List(0).AsReadOnly()); _session = session ?? throw new ArgumentNullException(nameof(session)); _rowLimit = 0; _ignoreProfileEvents = true; _reinterpretedColumnsCache = Array.Empty(); _recordsAffected = 0; Debug.Assert(session.IsDisposed); State = ClickHouseDataReaderState.Closed; } // Note that this xml comment is inherited by ClickHouseColumnWriter.ConfigureColumn /// /// Applies the settings to the specified column. /// /// The name of the column. /// The settings. public void ConfigureColumn(string name, ClickHouseColumnSettings columnSettings) { var index = GetOrdinal(name); if (index < 0) throw new ArgumentException($"A column with the name \"{name}\" not found.", nameof(name)); ConfigureColumn(index, columnSettings); } // Note that this xml comment is inherited by ClickHouseColumnWriter.ConfigureColumn /// /// Applies the settings to the specified column. /// /// The zero-based column ordinal. /// The settings. public void ConfigureColumn(int ordinal, ClickHouseColumnSettings columnSettings) { if (_rowIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be reconfigured during reading."); if (State == ClickHouseDataReaderState.ProfileEvents) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be configured when reading profile events."); IClickHouseColumnReinterpreter? reinterpreter = null; if (columnSettings.ColumnType != null) reinterpreter = ClickHouseColumnReinterpreter.Create(columnSettings.ColumnType); var columnName = _currentTable.Header.Columns[ordinal].Name; if (_columnSettings != null) { _columnSettings[ordinal] = _columnSettings[ordinal].WithColumnSettings(columnName, columnSettings, reinterpreter); } else { _columnSettings = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; _columnSettings[ordinal] = new ClickHouseReaderColumnSettings(columnSettings, reinterpreter); } } /// /// Configures the reader to invoke an arbitrary type conversion callback function when reading a value of a column. /// /// The type supported by ClickHouseDataReader. The column is expected to be convertible to this type. /// The type to which column values must be converted. /// The name of the column. /// The callback function converting a value of type to type . /// The callback function () must never return when its argument is not null. public void ConfigureColumnReader(string name, Func? readAs) { var index = GetOrdinal(name); if (index < 0) throw new ArgumentException($"A column with the name \"{name}\" not found.", nameof(name)); ConfigureColumnReader(index, readAs); } /// /// Configures the reader to invoke an arbitrary type conversion callback function when reading a value of a column. /// /// The type supported by ClickHouseDataReader. The column is expected to be convertible to this type. /// The type to which column values must be converted. /// The zero-based column ordinal. /// The callback function converting a value of type to type . /// The callback function () must never return when its argument is not null. public void ConfigureColumnReader(int ordinal, Func? readAs) { if (_rowIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be reconfigured during reading."); if (State == ClickHouseDataReaderState.ProfileEvents) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be configured when reading profile events."); if (readAs == null && _columnSettings == null) return; IClickHouseColumnReinterpreter? reinterpreter = null; if (readAs != null) { if (typeof(T) == typeof(object)) reinterpreter = new ClickHouseObjectColumnReinterpreter((Func)(object)readAs); else reinterpreter = ClickHouseColumnReinterpreter.Create(readAs); // Dry run. The interpreter could invoke the function 'readAs' and it could fail. reinterpreter.TryReinterpret(_currentTable.Columns[ordinal]); } if (_columnSettings == null) _columnSettings = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; _columnSettings[ordinal] = _columnSettings[ordinal].WithUserDefinedReader(_currentTable.Header.Columns[ordinal].Name, reinterpreter); } /// /// Configures the reader to invoke an arbitrary type converter callback function provided by the dispatcher when reading a value of a column. /// /// The zero-based column ordinal. /// The instance of a dispatcher that provides a type converter callback function or to reset the dispatcher for the column. /// A callback function provided by the dispathcer must never return when its argument is not null. public void ConfigureColumnReader(int ordinal, IConverterDispatcher? readAsDispatcher) { if (_rowIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be reconfigured during reading."); if (State == ClickHouseDataReaderState.ProfileEvents) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be configured when reading profile events."); if (readAsDispatcher == null && _columnSettings == null) return; IClickHouseColumnReinterpreter? reinterpreter = null; if (readAsDispatcher != null) { var dispatcher = ClickHouseColumnReinterpreter.CreateDispatcher(readAsDispatcher); if (!_currentTable.Columns[ordinal].TryDipatch(dispatcher, out reinterpreter)) reinterpreter = dispatcher.Dispatch(); // Dry run. The interpreter could invoke the function 'readAs' and it could fail. reinterpreter?.TryReinterpret(_currentTable.Columns[ordinal]); } if (reinterpreter == null && _columnSettings == null) return; if (_columnSettings == null) _columnSettings = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; _columnSettings[ordinal] = _columnSettings[ordinal].WithUserDefinedReader(_currentTable.Header.Columns[ordinal].Name, reinterpreter); } // Note that this xml comment is inherited by ClickHouseColumnWriter.ConfigureColumnWriter /// /// Applies the settings to all columns. All previously applied settings are discarded. /// /// The settings. public void ConfigureDataReader(ClickHouseColumnSettings columnSettings) { if (_rowIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The reader can't be reconfigured during reading."); IClickHouseColumnReinterpreter? reinterpreter = null; if (columnSettings.ColumnType != null) reinterpreter = ClickHouseColumnReinterpreter.Create(columnSettings.ColumnType); if (_columnSettings == null) { var settings = new ClickHouseReaderColumnSettings(columnSettings, reinterpreter); _columnSettings = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; for (int i = 0; i < _columnSettings.Length; i++) _columnSettings[i] = settings; } else { var settingsCopy = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; for (int i = 0; i < settingsCopy.Length; i++) settingsCopy[i] = _columnSettings[i].WithColumnSettings(_currentTable.Header.Columns[i].Name, columnSettings, reinterpreter); _columnSettings = settingsCopy; } } /// /// Configures the reader to invoke an arbitrary type converter callback function provided by the dispatcher when reading values. /// /// The instance of a dispatcher that provides a type converter callback function. /// A callback function provided by the dispathcer must never return when its argument is not null. public void ConfigureDataReader(IConverterDispatcher readAsDispatcher) { if (readAsDispatcher == null) throw new ArgumentNullException(nameof(readAsDispatcher)); if (_rowIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be reconfigured during reading."); if (State == ClickHouseDataReaderState.ProfileEvents) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The column can't be configured when reading profile events."); ClickHouseReaderColumnSettings[] columnSettings; if (_columnSettings == null) columnSettings = new ClickHouseReaderColumnSettings[_currentTable.Columns.Count]; else columnSettings = _columnSettings.ToArray(); Debug.Assert(columnSettings.Length == FieldCount); var dispatcher = ClickHouseColumnReinterpreter.CreateDispatcher(readAsDispatcher); bool affected = false; for (int ordinal = 0; ordinal < columnSettings.Length; ordinal++) { if (!_currentTable.Columns[ordinal].TryDipatch(dispatcher, out var reinterpreter)) reinterpreter = dispatcher.Dispatch(); if (!ReferenceEquals(reinterpreter, columnSettings[ordinal].Reinterpreter)) continue; // Dry run. The interpreter could invoke the function 'readAs' and it could fail. reinterpreter?.TryReinterpret(_currentTable.Columns[ordinal]); columnSettings[ordinal] = columnSettings[ordinal].WithUserDefinedReader(_currentTable.Header.Columns[ordinal].Name, reinterpreter); affected = true; } if (affected) _columnSettings = columnSettings; } // Note that this xml comment is inherited by ClickHouseColumnWriter.GetFieldTypeInfo /// /// Gets the which represents information about the type of the column. /// /// The zero-based column ordinal. /// The which represents information about the type of the column. public IClickHouseTypeInfo GetFieldTypeInfo(int ordinal) { return _currentTable.Header.Columns[ordinal].TypeInfo; } // Note that this xml comment is inherited by ClickHouseColumnWriter.GetName /// /// Gets the name of the specified column. /// /// The zero-based column ordinal. /// The name of the specified column. public sealed override string GetName(int ordinal) { return _currentTable.Header.Columns[ordinal].Name; } // Note that this xml comment is inherited by ClickHouseColumnWriter.GetDataTypeName /// /// Gets the name of the data type of the specified column. /// /// The zero-based column ordinal. /// The string representing the data type of the specified column. public sealed override string GetDataTypeName(int ordinal) { return _currentTable.Header.Columns[ordinal].TypeInfo.ComplexTypeName; } // Note that this xml comment is inherited by ClickHouseColumnWriter.GetFieldType /// /// Gets the that is the data type of the object. /// /// The zero-based column ordinal. /// The that is the data type of the object. public override Type GetFieldType(int ordinal) { // This method must return the type of a value returned by GetValue. // GetValue should always return DBNull.Value instead of null. // So an actual field type should be unboxed from Nullable. var reinterpreter = _columnSettings?[ordinal].Reinterpreter; var type = reinterpreter?.ExternalConvertToType ?? reinterpreter?.BuiltInConvertToType; type ??= _currentTable.Header.Columns[ordinal].TypeInfo.GetFieldType(); return Nullable.GetUnderlyingType(type) ?? type; } // Note that this xml comment is inherited by ClickHouseColumnWriter.GetOrdinal /// /// Gets the column ordinal, given the name of the column. /// /// The name of the column. /// The zero-based column ordinal. public sealed override int GetOrdinal(string name) { if (name == null) throw new ArgumentNullException(nameof(name)); return CommonUtils.GetColumnIndex(_currentTable.Header.Columns, name); } /// /// Reads the value as an array of and copies values from it to the buffer. /// /// The zero-based column ordinal. /// The index within the field from which to begin the read operation. /// The buffer into which to copy bytes. /// The index within the where the write operation is to start. /// The maximum length to copy into the buffer. /// The actual number of bytes copied. public sealed override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) { var arrayColumn = _reinterpretedColumnsCache[ordinal] as IClickHouseArrayTableColumn; if (arrayColumn == null) { arrayColumn = _currentTable.Columns[ordinal].TryReinterpretAsArray(); if (arrayColumn != null) _reinterpretedColumnsCache[ordinal] = arrayColumn; } if (arrayColumn != null) { CheckRowIndex(); return arrayColumn.CopyTo(_rowIndex, new Span(buffer, bufferOffset, Math.Min(buffer?.Length ?? 0, length)), checked((int)dataOffset)); } var value = GetFieldValue(ordinal, null); if (value == null) { if (dataOffset == 0) return 0; throw new ArgumentOutOfRangeException(nameof(dataOffset)); } ReadOnlySpan source = value; Span target = buffer; int resultLength = Math.Min(value.Length - (int) dataOffset, length); source = source.Slice((int) dataOffset, resultLength); target = target.Slice(bufferOffset, resultLength); source.CopyTo(target); return resultLength; } /// /// Reads the value as a and copies characters from it to the buffer. /// /// The zero-based column ordinal. /// The index within the field from which to begin the read operation. /// The buffer into which to copy characters of the string. /// The index within the where the write operation is to start. /// The maximum length to copy into the buffer. /// The actual number of characters copied. public sealed override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) { var arrayColumn = _reinterpretedColumnsCache[ordinal] as IClickHouseArrayTableColumn; if (arrayColumn == null) { arrayColumn = _currentTable.Columns[ordinal].TryReinterpretAsArray(); if (arrayColumn != null) _reinterpretedColumnsCache[ordinal] = arrayColumn; } if (arrayColumn != null) { CheckRowIndex(); return arrayColumn.CopyTo(_rowIndex, new Span(buffer, bufferOffset, Math.Min(buffer?.Length ?? 0, length)), checked((int)dataOffset)); } var value = GetFieldValue(ordinal, null); if (value == null) { if (dataOffset == 0) return 0; throw new ArgumentOutOfRangeException(nameof(dataOffset)); } ReadOnlySpan source = value; Span target = buffer; int resultLength = Math.Min(value.Length - (int) dataOffset, length); source = source.Slice((int) dataOffset, resultLength); target = target.Slice(bufferOffset, resultLength); source.CopyTo(target); return resultLength; } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public BigInteger GetBigInteger(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override bool GetBoolean(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override byte GetByte(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sbyte GetSByte(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override char GetChar(int ordinal) { return GetFieldValue(ordinal); } #if NET6_0_OR_GREATER /// /// Gets the value of the specified column as a object. /// /// The zero-based column ordinal. /// The value of the specified column. public DateOnly GetDate(int ordinal) { return GetFieldValue(ordinal); } #endif /// /// Gets the value of the specified column as a object. /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override DateTime GetDateTime(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override decimal GetDecimal(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override double GetDouble(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override float GetFloat(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a globally unique identifier (). /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override Guid GetGuid(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override short GetInt16(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public ushort GetUInt16(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override int GetInt32(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an . /// /// The zero-based column ordinal. /// The value of the specified column. public uint GetUInt32(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override long GetInt64(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as a . /// /// The zero-based column ordinal. /// The value of the specified column. public ulong GetUInt64(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an instance of . /// /// The zero-based column ordinal. /// The value of the specified column. public IPAddress GetIPAddress(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an instance of . /// /// The zero-based column ordinal. /// The value of the specified column. public sealed override string GetString(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an instance of . /// /// The zero-based column ordinal. /// The default value which should be returned if the value of the specified column is . /// The value of the specified column or if the value of the column is . [return: NotNullIfNotNull("nullValue")] public string? GetString(int ordinal, string? nullValue) { return GetFieldValue(ordinal, nullValue); } /// /// Gets the value of the specified column as a object. /// /// The zero-based column ordinal. /// The value of the specified column. public DateTimeOffset GetDateTimeOffset(int ordinal) { return GetFieldValue(ordinal); } /// /// Gets the value of the specified column as an object of the type . /// /// The expected type of the column's value. /// The zero-based column ordinal. /// The default value which should be returned if the value of the specified column is . /// The value of the specified column or if the value of the column is . [return: NotNullIfNotNull("nullValue")] public T? GetFieldValue(int ordinal, T? nullValue) where T : class { CheckRowIndex(); var column = _currentTable.Columns[ordinal]; if (column is IClickHouseTableColumn typedColumn) return typedColumn.GetValue(_rowIndex) ?? nullValue; if (_reinterpretedColumnsCache[ordinal] is IClickHouseTableColumn reinterpretedColumn) return reinterpretedColumn.GetValue(_rowIndex) ?? nullValue; var rc = column.TryReinterpret(); if (rc != null) { _reinterpretedColumnsCache[ordinal] = rc; return rc.GetValue(_rowIndex) ?? nullValue; } if (column.IsNull(_rowIndex)) return nullValue; var value = column.GetValue(_rowIndex); return (T) value; } /// [return: NotNullIfNotNull("nullValue")] public T? GetFieldValue(int ordinal, T? nullValue) where T : struct { CheckRowIndex(); var column = _currentTable.Columns[ordinal]; if (column is IClickHouseTableColumn nullableTypedColumn) return nullableTypedColumn.GetValue(_rowIndex) ?? nullValue; var rc = _reinterpretedColumnsCache[ordinal] as IClickHouseTableColumn; if (rc != null) return rc.GetValue(_rowIndex) ?? nullValue; rc = column.TryReinterpret(); if (rc == null) { var structRc = column.TryReinterpret(); if (structRc != null) rc = new NullableStructTableColumn(null, structRc); } if (rc == null && column is IClickHouseTableColumn typedColumn) rc = new NullableStructTableColumn(null, typedColumn); if (rc != null) { _reinterpretedColumnsCache[ordinal] = rc; return rc.GetValue(_rowIndex) ?? nullValue; } if (column.IsNull(_rowIndex)) return nullValue; var value = column.GetValue(_rowIndex); return (T) value; } /// /// Gets the value of the specified column as an object of the type . /// /// The expected type of the column's value. /// The zero-based column ordinal. /// The value of the specified column. public sealed override T GetFieldValue(int ordinal) { CheckRowIndex(); var column = _currentTable.Columns[ordinal]; if (column.IsNull(_rowIndex)) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The value is null."); if (column is IClickHouseTableColumn typedColumn) return typedColumn.GetValue(_rowIndex); if (_reinterpretedColumnsCache[ordinal] is IClickHouseTableColumn reinterpretedColumn) return reinterpretedColumn.GetValue(_rowIndex); var rc = column.TryReinterpret(); if (rc != null) { _reinterpretedColumnsCache[ordinal] = rc; return rc.GetValue(_rowIndex); } var value = column.GetValue(_rowIndex); return (T) value; } /// /// Gets the value of the specified column as an . /// /// The zero-based column ordinal. /// The value of the specified column or . public sealed override object GetValue(int ordinal) { CheckRowIndex(); var column = _currentTable.Columns[ordinal]; return column.GetValue(_rowIndex); } /// /// Populates an array of objects with the column values of the current row. /// /// An array of into which to copy the attribute columns. /// The number of instances of in the array. public sealed override int GetValues(object[] values) { CheckRowIndex(); var count = Math.Min(_currentTable.Columns.Count, values.Length); for (int i = 0; i < count; i++) { var column = _currentTable.Columns[i]; values[i] = column.GetValue(_rowIndex); } return count; } /// /// Gets a value that indicates whether the column contains non-existent or missing values. /// /// The zero-based column ordinal. /// if the specified column is equivalent to . Otherwise . public sealed override bool IsDBNull(int ordinal) { CheckRowIndex(); var column = _currentTable.Columns[ordinal]; return column.IsNull(_rowIndex); } /// public sealed override object this[int ordinal] => GetValue(ordinal); /// /// Gets the value of the specified column as an . /// /// The name of the column. /// The value of the specified column or . public sealed override object this[string name] { get { var ordinal = GetOrdinal(name); if (ordinal < 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Column \"{name}\" not found."); return GetValue(ordinal); } } /// /// Advances the reader to the next record in a result set. /// /// if there are more rows or if there aren't. public sealed override bool Read() { return TaskHelper.WaitNonAsyncTask(Read(false, CancellationToken.None)); } /// /// Asyncronously advances the reader to the next record in a result set. /// /// /// A whose is /// if there are more rows or if there aren't. /// public new ValueTask ReadAsync() { return Read(true, CancellationToken.None); } /// /// Asyncronously advances the reader to the next record in a result set. /// /// The cancellation instruction. /// /// A whose is /// if there are more rows or if there aren't. /// public new ValueTask ReadAsync(CancellationToken cancellationToken) { return Read(true, cancellationToken); } /// protected sealed override async Task ReadAsyncInternal(CancellationToken cancellationToken) { return await Read(true, cancellationToken); } private async ValueTask Read(bool async, CancellationToken cancellationToken) { if (State == ClickHouseDataReaderState.Closed || State == ClickHouseDataReaderState.Broken || State == ClickHouseDataReaderState.ClosePending) return false; bool result; if (++_rowIndex >= _currentTable.Header.RowCount) { _rowIndex = _currentTable.Header.RowCount; if (State == ClickHouseDataReaderState.Data || State == ClickHouseDataReaderState.Totals || State == ClickHouseDataReaderState.Extremes) { result = await Read(false, async, cancellationToken); if (!result && _rowLimit == ClickHouseDataReaderRowLimit.OneResult && State == ClickHouseDataReaderState.NextResultPending) await Cancel(false, async); } else { result = false; } } else { result = true; } if (_rowLimit == ClickHouseDataReaderRowLimit.OneRow && result) await Cancel(false, async); return result; } private async ValueTask Read(bool nextResult, bool async, CancellationToken cancellationToken) { while (true) { ClickHouseTable nextTable; try { var message = _nextResultMessage; if (message == null) message = await _session.ReadMessage(async, cancellationToken); else _nextResultMessage = null; switch (message.MessageCode) { case ServerMessageCode.Data: switch (State) { case ClickHouseDataReaderState.NextResultPending: State = ClickHouseDataReaderState.Data; goto case ClickHouseDataReaderState.Data; case ClickHouseDataReaderState.Data: { var dataMessage = (ServerDataMessage) message; nextTable = await _session.ReadTable(dataMessage, _columnSettings, async, cancellationToken); break; } case ClickHouseDataReaderState.Totals: case ClickHouseDataReaderState.Extremes: { var dataMessage = (ServerDataMessage)message; var table = await _session.SkipTable(dataMessage, async, cancellationToken); if (table.RowCount != 0 || table.Columns.Count != 0) throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, "Unexpected data block after totals or extremes."); continue; } case ClickHouseDataReaderState.ProfileEvents: _nextResultMessage = message; State = ClickHouseDataReaderState.NextResultPending; return false; default: goto UNEXPECTED_RESPONSE; } break; case ServerMessageCode.Error: State = ClickHouseDataReaderState.Closed; await _session.Dispose(async); throw ((ServerErrorMessage) message).Exception; case ServerMessageCode.Progress: var progressMessage = (ServerProgressMessage) message; ExecutionProgress = progressMessage.ExecutionProgress; _recordsAffected = ExecutionProgress.Rows; continue; case ServerMessageCode.EndOfStream: State = ClickHouseDataReaderState.Closed; await _session.Dispose(async); return false; case ServerMessageCode.ProfileInfo: continue; case ServerMessageCode.Totals: switch (State) { case ClickHouseDataReaderState.NextResultPending: State = ClickHouseDataReaderState.Totals; goto case ClickHouseDataReaderState.Totals; case ClickHouseDataReaderState.Totals: var totalsMessage = (ServerDataMessage) message; nextTable = await _session.ReadTable(totalsMessage, _columnSettings, async, cancellationToken); break; case ClickHouseDataReaderState.Data: case ClickHouseDataReaderState.Extremes: case ClickHouseDataReaderState.ProfileEvents: _nextResultMessage = message; State = ClickHouseDataReaderState.NextResultPending; return false; default: goto UNEXPECTED_RESPONSE; } break; case ServerMessageCode.Extremes: switch (State) { case ClickHouseDataReaderState.NextResultPending: State = ClickHouseDataReaderState.Extremes; goto case ClickHouseDataReaderState.Extremes; case ClickHouseDataReaderState.Extremes: var extremesMessage = (ServerDataMessage) message; nextTable = await _session.ReadTable(extremesMessage, _columnSettings, async, cancellationToken); break; case ClickHouseDataReaderState.Data: case ClickHouseDataReaderState.Totals: case ClickHouseDataReaderState.ProfileEvents: _nextResultMessage = message; State = ClickHouseDataReaderState.NextResultPending; return false; default: goto UNEXPECTED_RESPONSE; } break; case ServerMessageCode.ProfileEvents: if (_ignoreProfileEvents) { await _session.SkipTable((ServerDataMessage)message, async, cancellationToken); continue; } switch (State) { case ClickHouseDataReaderState.NextResultPending: State = ClickHouseDataReaderState.ProfileEvents; goto case ClickHouseDataReaderState.ProfileEvents; case ClickHouseDataReaderState.ProfileEvents: var profileEventsMessage = (ServerDataMessage) message; nextTable = await _session.ReadTable(profileEventsMessage, null, async, cancellationToken); break; case ClickHouseDataReaderState.Data: case ClickHouseDataReaderState.Extremes: case ClickHouseDataReaderState.Totals: _nextResultMessage = message; State = ClickHouseDataReaderState.NextResultPending; return false; default: goto UNEXPECTED_RESPONSE; } break; case ServerMessageCode.Pong: case ServerMessageCode.Hello: case ServerMessageCode.Log: UNEXPECTED_RESPONSE: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message: \"{message.MessageCode}\"."); default: throw new NotSupportedException($"Internal error. Message code \"{message.MessageCode}\" not supported."); } } catch (ClickHouseHandledException) { throw; } catch (ClickHouseServerException) { throw; } catch (Exception ex) { State = ClickHouseDataReaderState.Broken; var aggrEx = await _session.SetFailed(ex, true, async); if (aggrEx != null) throw aggrEx; throw; } if (nextTable.Header.RowCount == 0) continue; _currentTable = nextTable; _reinterpretedColumnsCache = new IClickHouseTableColumn[_currentTable.Columns.Count]; _recordsAffected = checked(_recordsAffected + (ulong) nextTable.Header.RowCount); _rowIndex = nextResult ? -1 : 0; return true; } } /// /// Asyncronously advances the reader to the next result set. The ClickHouse server can send totals or extremes as additional result sets. /// /// A representing asyncronous operation. The result () is /// if there are more result sets; otherwise /// public override async Task NextResultAsync(CancellationToken cancellationToken) { return await NextResult(true, cancellationToken); } /// /// Advances the reader to the next result set. The ClickHouse server can send totals or extremes as additional result sets. /// /// if there are more result sets; otherwise public override bool NextResult() { return TaskHelper.WaitNonAsyncTask(NextResult(false, CancellationToken.None)); } private async ValueTask NextResult(bool async, CancellationToken cancellationToken) { if (State == ClickHouseDataReaderState.Data || State == ClickHouseDataReaderState.Totals || State == ClickHouseDataReaderState.Extremes || State == ClickHouseDataReaderState.ProfileEvents) { bool canReadNext; do { _rowIndex = Math.Max(_rowIndex, _currentTable.Header.RowCount - 1); // TODO: skip without actual reading canReadNext = await Read(false, async, cancellationToken); } while (canReadNext); } if (State != ClickHouseDataReaderState.NextResultPending) return false; return await Read(true, async, cancellationToken); } private async ValueTask Cancel(bool disposing, bool async) { if (State == ClickHouseDataReaderState.Closed || State == ClickHouseDataReaderState.Broken || State == ClickHouseDataReaderState.ClosePending) return; try { await _session.SendCancel(async); State = ClickHouseDataReaderState.ClosePending; } catch (Exception ex) { State = ClickHouseDataReaderState.Broken; await _session.SetFailed(ex, false, async); if (!disposing) throw; } } /// /// Closes the reader. /// public override void Close() { TaskHelper.WaitNonAsyncTask(Close(false, false)); } /// /// Asyncronously closes the reader. /// /// A representing the asyncronous operation. public override async Task CloseAsync() { await Close(false, true); } private async ValueTask Close(bool disposing, bool async) { if (_session.IsDisposed || _session.IsFailed) return; if (!(State == ClickHouseDataReaderState.Closed || State == ClickHouseDataReaderState.Broken)) { await Cancel(disposing, async); try { while (true) { var message = _nextResultMessage ?? await _session.ReadMessage(async, CancellationToken.None); _nextResultMessage = null; switch (message.MessageCode) { case ServerMessageCode.Data: case ServerMessageCode.Totals: case ServerMessageCode.Extremes: case ServerMessageCode.ProfileEvents: var dataMessage = (ServerDataMessage) message; await _session.SkipTable(dataMessage, async, CancellationToken.None); continue; case ServerMessageCode.Error: State = ClickHouseDataReaderState.Closed; if (disposing) break; throw ((ServerErrorMessage) message).Exception; case ServerMessageCode.Progress: var progressMessage = (ServerProgressMessage) message; ExecutionProgress = progressMessage.ExecutionProgress; _recordsAffected = ExecutionProgress.Rows; continue; case ServerMessageCode.EndOfStream: State = ClickHouseDataReaderState.Closed; break; case ServerMessageCode.ProfileInfo: continue; case ServerMessageCode.Pong: case ServerMessageCode.Hello: case ServerMessageCode.Log: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Unexpected server message: \"{message.MessageCode}\"."); default: throw new NotSupportedException($"Internal error. Message code \"{message.MessageCode}\" not supported."); } break; } } catch (ClickHouseHandledException ex) { if (!disposing) throw; State = ClickHouseDataReaderState.Broken; await _session.SetFailed(ex.InnerException, false, async); return; } catch (Exception ex) { State = ClickHouseDataReaderState.Broken; var aggrEx = await _session.SetFailed(ex, false, async); if (disposing) return; if (aggrEx != null) throw aggrEx; throw; } } await _session.Dispose(async); } /// protected override void Dispose(bool disposing) { if (!disposing) return; TaskHelper.WaitNonAsyncTask(Close(true, false)); } /// public override ValueTask DisposeAsync() { return Close(true, true); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void CheckRowIndex() { if (_rowIndex >= _currentTable.Header.RowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "The reader is reached the end of the table."); if (_rowIndex < 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"There are no rows to read. The call of the method {nameof(Read)} is required."); } /// /// Not supported. An enumerator iterating through the rows of the reader is not implemented. /// /// Always throws . public override IEnumerator GetEnumerator() { throw new NotImplementedException(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDataReaderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Data.Common; using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient { /// /// This is an infrastracture class. It allows it's descendants to overload the method . /// public abstract class ClickHouseDataReaderBase : DbDataReader { private protected ClickHouseDataReaderBase() { } /// public sealed override Task ReadAsync(CancellationToken cancellationToken) { return ReadAsyncInternal(cancellationToken); } /// /// When overriden in a derived class should asyncronously advance the reader to the next record in a result set. /// /// The cancellation instruction. /// /// A representing asyncronous operation. The result () is /// if there are more rows or if there aren't. /// protected abstract Task ReadAsyncInternal(CancellationToken cancellationToken); } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDataReaderRowLimit.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { internal enum ClickHouseDataReaderRowLimit { Zero = 0, OneRow = 1, OneResult = 2, Infinite = 3 } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDataReaderState.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// Describes the current state of the . /// public enum ClickHouseDataReaderState { /// /// The reader was broken. /// This usualy indicates that there were a network error while fetching data. /// Broken = 0, /// /// The reader is closed. Which means that the end of the data stream was reached. /// Closed = 1, /// /// The reader is currently reading the main result set. /// Data = 2, /// /// The reader reached to the end of the current result set and there possible are the next result set. ///
/// Use one of methods or /// to check if there are the next result set. /// NextResultPending = 3, /// /// The reader is currently reading the TOTALS result set. /// Totals = 4, /// /// The reader is currently reading the EXTREMES result set. /// Extremes = 5, /// /// The reader can no longer read the data and waiting for closing. ///
/// Use one of methods /// to close the reader. /// /// /// This state usualy indicates that the reader was created with a that /// forbids further reading even if there are more rows in the current result set or there are more result sets. /// ClosePending = 6, /// /// The reader is currently reading profile events. /// ProfileEvents = 7 } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDbProviderFactory.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Data.Common; namespace Octonica.ClickHouseClient { /// /// Represents a set of methods for creating instances of ClickHouseClient's implementation of the data source classes. /// public class ClickHouseDbProviderFactory : DbProviderFactory { /// /// Returns a new instance of . /// /// A new instance of . public override DbConnectionStringBuilder CreateConnectionStringBuilder() { return new ClickHouseConnectionStringBuilder(); } /// /// Returns a new instance of . /// /// A new instance of . public override DbConnection CreateConnection() { return new ClickHouseConnection(); } /// /// Returns a new instance of . /// /// A new instance of . public override DbCommand CreateCommand() { return new ClickHouseCommand(); } /// /// Returns a new instance of . /// /// A new instance of . public override DbParameter CreateParameter() { return new ClickHouseParameter(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseDbType.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Data; namespace Octonica.ClickHouseClient { /// /// Specifies the data type of a field (column) or a object. /// public enum ClickHouseDbType { /// /// The type is not supported by the client. An encoding should be defined explicitly via /// or . /// /// This value corresponds to . AnsiString = DbType.AnsiString, /// /// An array of bytes. /// /// This value corresponds to . Binary = DbType.Binary, /// /// An 8-bit unsigned integer ranging in value from 0 to 255. /// /// This value corresponds to . Byte = DbType.Byte, /// /// A simple type representing Boolean values of or . /// /// This value corresponds to . Boolean = DbType.Boolean, /// /// The ClickHouse type Decimal(18, 4). /// /// This value corresponds to . Currency = DbType.Currency, /// /// A type representing a date value without a time. /// /// This value corresponds to . Date = DbType.Date, /// /// A type representing a date and time value. /// /// This value corresponds to . DateTime = DbType.DateTime, /// /// The ClickHouse type Decimal(38, 9). /// /// This value corresponds to . Decimal = DbType.Decimal, /// /// This value corresponds to . Double = DbType.Double, /// /// The ClickHouse type UUID. /// /// This value corresponds to . Guid = DbType.Guid, /// /// This value corresponds to . Int16 = DbType.Int16, /// /// This value corresponds to . Int32 = DbType.Int32, /// /// This value corresponds to . Int64 = DbType.Int64, /// /// A general type representing a value of either an unknown type or the ClickHouse type Nothing. /// /// This value corresponds to . Object = DbType.Object, /// /// This value corresponds to . SByte = DbType.SByte, /// /// This value corresponds to . Single = DbType.Single, /// /// A variable-length string. /// /// This value corresponds to . String = DbType.String, /// /// The type is not supported by the client. /// /// This value corresponds to . Time = DbType.Time, /// /// This value corresponds to . UInt16 = DbType.UInt16, /// /// This value corresponds to . UInt32 = DbType.UInt32, /// /// This value corresponds to . UInt64 = DbType.UInt64, /// /// A type representing numeric values with the specified precision (from 1 to 76) and scale (from 0 to the precision). /// /// This value corresponds to . VarNumeric = DbType.VarNumeric, /// /// The type is not supported by the client. /// /// This value corresponds to . AnsiStringFixedLength = DbType.AnsiStringFixedLength, /// /// A fixed-length string. /// /// This value corresponds to . StringFixedLength = DbType.StringFixedLength, /// /// The type is not supported by the client. /// /// This value corresponds to . Xml = DbType.Xml, /// /// with an accuracy of 100 nanoseconds. /// /// This value corresponds to . DateTime2 = DbType.DateTime2, /// /// A type representing a date and time value with time zone awareness. /// /// This value corresponds to . DateTimeOffset = DbType.DateTimeOffset, /// /// It's not a valid code for any type. This value is used as the delimiter between and ClickHouse-specific type codes. /// ClickHouseSpecificTypeDelimiterCode = 0x3FFF, /// /// A type representing an IP v4 address. /// IpV4 = ClickHouseSpecificTypeDelimiterCode + 1, /// /// A type representing an IP v6 address. /// IpV6 = ClickHouseSpecificTypeDelimiterCode + 2, /// /// A type representing a variable-length array of values. /// Array = ClickHouseSpecificTypeDelimiterCode + 3, /// /// A type representing a fixed-length set of values. /// Tuple = ClickHouseSpecificTypeDelimiterCode + 4, /// /// A type representing NULL value. /// Nothing = ClickHouseSpecificTypeDelimiterCode + 5, /// /// A type representing Enum value. /// Enum = ClickHouseSpecificTypeDelimiterCode + 6, /// /// A type representing a date and time value with defined sub-second precision. /// Supported range of values: [1900-01-01 00:00:00, 2299-12-31 23:59:59.99999999] /// DateTime64 = ClickHouseSpecificTypeDelimiterCode + 7, /// /// The ClickHouse type Map(key, value). This type is not supported by . /// Map = ClickHouseSpecificTypeDelimiterCode + 8, /// /// An integral type representing signed 128-bit integers with values between -170141183460469231731687303715884105728 and 170141183460469231731687303715884105727. /// Int128 = ClickHouseSpecificTypeDelimiterCode + 9, /// /// An integral type representing unsigned 128-bit integers with values between 0 and 340282366920938463463374607431768211455. /// UInt128 = ClickHouseSpecificTypeDelimiterCode + 10, /// /// An integral type representing signed 256-bit integers with values between -57896044618658097711785492504343953926634992332820282019728792003956564819968 and 57896044618658097711785492504343953926634992332820282019728792003956564819967. /// Int256 = ClickHouseSpecificTypeDelimiterCode + 11, /// /// An integral type representing unsigned 256-bit integers with values between 0 and 115792089237316195423570985008687907853269984665640564039457584007913129639935. /// UInt256 = ClickHouseSpecificTypeDelimiterCode + 12, /// /// A type representing a date value without a time. Supports the date range same with . /// Stored in four bytes as the number of days since 1970-01-01. /// Date32 = ClickHouseSpecificTypeDelimiterCode + 13, /// /// This type represents a union of other data types. Type Variant(T1, T2, ..., TN) means that each row of this type /// has a value of either type T1 or T2 or ... or TN or none of them (NULL value). /// Variant = ClickHouseSpecificTypeDelimiterCode + 14, } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseFlushMode.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading; namespace Octonica.ClickHouseClient { /// /// Describes a set of strategies used by for commiting a transaction. /// public enum ClickHouseTransactionMode { /// /// The default strategy. This strategy is used if no strategy is specified. The same as . /// Default = 0, /// /// will send a table and finish the query (commits transaction). /// Next table (if any) will be sent with the new INSERT query (in a new transaction). /// Auto = 1, /// /// will not send confirmation after writing a table. /// Next table (if any) will be sent with the current INSERT query (in the same transaction). /// /// Call the method (or ) after writing tables. Manual = 2, /// /// will send each block of data in a separate query (one transaction per block). /// Block = 3 } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseParameter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Data; using System.Data.Common; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Net; using System.Net.Sockets; using System.Text; using System.Text.RegularExpressions; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { /// /// Represents a parameter to a . This class cannot be inherited. /// public sealed class ClickHouseParameter : DbParameter, ICloneable { // https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/query_language/syntax.md private static readonly Regex ParameterNameRegex = new Regex("^[a-zA-Z_][0-9a-zA-Z_]*$"); private string _parameterName; private object? _value; private int _size; private TimeZoneInfo? _timeZone; private bool? _forcedNullable; private ClickHouseDbType? _forcedType; private byte? _forcedScale; private byte? _forcedPrecision; private int? _forcedArrayRank; private IntermediateClickHouseTypeInfo? _valueTypeInfo; private string? _sourceColumn; internal string Id { get; private set; } /// /// Gets the collection to which this parameter is attached. /// public ClickHouseParameterCollection? Collection { get; internal set; } /// /// Gets or sets the of the parameter. /// /// One of the values. The default value is defined based on the type of the parameter's value. public ClickHouseDbType ClickHouseDbType { get => _forcedType ?? GetTypeFromValue().DbType; set => _forcedType = value; } /// /// Gets or sets the of the parameter. /// /// /// One of the values. The default value is defined based on the type of the parameter's value. /// For ClickHouse-specific types returns . /// public override DbType DbType { get { var chType = ClickHouseDbType; if (chType > ClickHouseDbType.ClickHouseSpecificTypeDelimiterCode) return DbType.Object; return (DbType) chType; } set => ClickHouseDbType = (ClickHouseDbType) value; } /// /// Gets the direction of the parameter. Always returns . /// /// Always returns . /// Throws on attempt to set the value different from . public override ParameterDirection Direction { get => ParameterDirection.Input; set { if (value != ParameterDirection.Input) throw new NotSupportedException("Only input parameters are supported."); } } /// /// Gets or sets the value indicating whether the type of the parameter is nullable. /// /// if the parameter's value can be NULL; otherwise . The default value is defined based on the type of the parameter's value. public override bool IsNullable { get => _forcedNullable ?? GetTypeFromValue().IsNullable; set => _forcedNullable = value; } /// /// Gets or sets the name of the parameter. /// [AllowNull] public sealed override string ParameterName { get => _parameterName; set { var id = GetId(value); Debug.Assert(value != null); if (StringComparer.Ordinal.Equals(id, Id)) { _parameterName = value; return; } if (Collection == null) { Id = id; _parameterName = value; return; } var oldId = Id; var oldParameterName = _parameterName; Id = id; _parameterName = value; try { Collection.OnParameterIdChanged(oldId, this); } catch { Id = oldId; _parameterName = oldParameterName; throw; } } } /// /// This property is ignored by ClickHouseClient. [AllowNull] public override string SourceColumn { get => _sourceColumn ?? string.Empty; set => _sourceColumn = value; } /// public override object? Value { get => _value; set { _value = value; _valueTypeInfo = null; } } /// /// This property is ignored by ClickHouseClient. public override bool SourceColumnNullMapping { get; set; } /// /// Gets or sets the size. This value is applied to the ClickHouse type FixedString. /// public override int Size { get => _size; set { _size = value; _valueTypeInfo = null; } } /// /// Gets or sets the precision. This value is applied to ClickHouse types Decimal and DateTime64. /// public override byte Precision { get => _forcedPrecision ?? (ClickHouseDbType == ClickHouseDbType.DateTime64 ? (byte) DateTime64TypeInfo.DefaultPrecision : DecimalTypeInfoBase.DefaultPrecision); set => _forcedPrecision = value; } /// /// Gets or sets the scale. This value is applied to the ClickHouse type Decimal. /// public override byte Scale { get => _forcedScale ?? DecimalTypeInfoBase.DefaultScale; set => _forcedScale = value; } /// /// Gets or sets the encoding that will be used when writing a string value to the database. /// public Encoding? StringEncoding { get; set; } /// /// This property allows to specify the timezone for datetime types (, /// , /// and ). /// public TimeZoneInfo? TimeZone { get => _timeZone; set { _timeZone = value; _valueTypeInfo = null; } } /// /// Gets or sets the value indicating whether the type is an array. /// /// if the value is an array; otherwise . The default value is defined based on the . public bool IsArray { get => ArrayRank > 0; set { if (value) { if (_forcedArrayRank == null || ArrayRank == 0) ArrayRank = 1; } else { if (_forcedArrayRank == null || ArrayRank > 0) ArrayRank = 0; } } } /// /// Gets or sets the rank (a number of dimensions) of an array. /// /// The rank of an array. 0 if the type is not an array. The default value is defined based on the type of the parameter's value. public int ArrayRank { get { if (_forcedArrayRank != null) return _forcedArrayRank.Value; var typeInfo = GetTypeFromValue(); if (typeInfo.ArrayRank > 0 && typeInfo.DbType == ClickHouseDbType.Byte && (_forcedType == ClickHouseDbType.String || _forcedType == ClickHouseDbType.StringFixedLength)) return typeInfo.ArrayRank - 1; return typeInfo.ArrayRank; } set { if (value < 0) throw new ArgumentException("The rank of an array must be a non-negative number.", nameof(value)); _forcedArrayRank = value; } } /// /// Gets or sets the mode of passing this parameter to the query. The value of this property overrides . /// /// The mode of passing this parameter to the query. The default value is . public ClickHouseParameterMode ParameterMode { get; set; } = ClickHouseParameterMode.Inherit; /// /// Initializes a new instance of with the default name. /// public ClickHouseParameter() : this("parameter") { } /// /// Initializes a new instance of with the specified name. /// /// The name of the parameter. public ClickHouseParameter(string parameterName) { if (parameterName == null) throw new ArgumentNullException(nameof(parameterName)); Id = GetId(parameterName); _parameterName = parameterName; } /// public override void ResetDbType() { _forcedType = null; _forcedNullable = null; _forcedPrecision = null; _forcedScale = null; _forcedArrayRank = null; _size = 0; StringEncoding = null; _timeZone = null; _valueTypeInfo = null; } object ICloneable.Clone() { return Clone(); } /// /// Creates a new that is a copy of this instance. The new parameter is not attached to any . /// /// A new that is a copy of this instance. public ClickHouseParameter Clone() { var result = new ClickHouseParameter(ParameterName); CopyTo(result); return result; } /// /// Copies all properties except of this parameter to the target parameter. /// /// The parameter to which the properties of this parameters should be copied. public void CopyTo(ClickHouseParameter parameter) { parameter._forcedType = _forcedType; parameter._forcedArrayRank = _forcedArrayRank; parameter._forcedNullable = _forcedNullable; parameter._forcedPrecision = _forcedPrecision; parameter._forcedScale = _forcedScale; parameter._forcedArrayRank = _forcedArrayRank; parameter._size = _size; parameter.StringEncoding = StringEncoding; parameter._timeZone = _timeZone; parameter._value = _value; parameter.SourceColumn = SourceColumn; parameter.SourceColumnNullMapping = SourceColumnNullMapping; parameter.SourceVersion = SourceVersion; parameter._valueTypeInfo = null; parameter.ParameterMode = ParameterMode; } internal ClickHouseParameterWriter CreateParameterWriter(IClickHouseTypeInfoProvider typeInfoProvider) { return CreateParameterWriter(typeInfoProvider, Create); static ClickHouseParameterWriter Create(string id, object? value, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo typeInfo, Type clrType) { return ClickHouseParameterWriter.Dispatch(typeInfo, value); } } internal IClickHouseColumnWriter CreateParameterColumnWriter(IClickHouseTypeInfoProvider typeInfoProvider) { return CreateParameterWriter(typeInfoProvider, Create); static IClickHouseColumnWriter Create(string id, object? value, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo typeInfo, Type clrType) { var columnBuilder = new ParameterColumnWriterBuilder(id, value, columnSettings, typeInfo); var column = TypeDispatcher.Dispatch(clrType, columnBuilder); return column; } } private T CreateParameterWriter(IClickHouseTypeInfoProvider typeInfoProvider, Func createWriter) { bool isNull = Value == DBNull.Value || Value == null; if (isNull && _forcedNullable == false) throw new ClickHouseException(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"The parameter \"{ParameterName}\" is declared as non-nullable but it's value is null."); var typeInfo = GetTypeInfo(typeInfoProvider); object? preparedValue = null; if (_forcedType == ClickHouseDbType.StringFixedLength) { Debug.Assert(typeInfo.TypeName == "FixedString"); ReadOnlySpan strSpan = default; bool isStr = false; if (Value is string strValue) { strSpan = strValue; isStr = true; } else if (Value is Memory mem) { strSpan = mem.Span; isStr = true; } else if (Value is ReadOnlyMemory roMem) { strSpan = roMem.Span; isStr = true; } if (isStr) { var encoding = StringEncoding ?? Encoding.UTF8; var size = encoding.GetByteCount(strSpan); if (size > Size) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"Parameter \"{ParameterName}\". The length of the string in bytes with encoding \"{encoding.EncodingName}\" is greater than the size of the parameter."); } var bytes = new byte[size]; encoding.GetBytes(strSpan, bytes); preparedValue = bytes; } } var clrType = isNull ? typeInfo.GetFieldType() : (preparedValue ?? Value)!.GetType(); var columnSettings = StringEncoding == null ? null : new ClickHouseColumnSettings(StringEncoding); return createWriter(Id, isNull ? null : preparedValue ?? Value, columnSettings, typeInfo, clrType); } internal IClickHouseColumnTypeInfo GetTypeInfo(IClickHouseTypeInfoProvider typeInfoProvider) { var adapter = new ParameterColumnTypeDescriptorAdapter(this); try { return typeInfoProvider.GetTypeInfo(adapter); } catch (ClickHouseException ex) { throw new ClickHouseException(ex.ErrorCode, $"Parameter \"{ParameterName}\". {ex.Message}", ex); } } private IntermediateClickHouseTypeInfo GetTypeFromValue() { if (_valueTypeInfo != null) return _valueTypeInfo.Value; var result = GetValueDependentType(); if (result == null) { try { result = ClickHouseTypeInfoProvider.GetTypeFromValue(Value?.GetType() ?? typeof(DBNull), Value == null, TimeZone); } catch (ClickHouseException ex) { throw new ClickHouseException(ex.ErrorCode, $"Parameter \"{ParameterName}\". {ex.Message}", ex); } } _valueTypeInfo = result; return result.Value; } private IntermediateClickHouseTypeInfo? GetValueDependentType() { if (Value is IPAddress ipAddress) { if (ipAddress.AddressFamily == AddressFamily.InterNetwork) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.IpV4, "IPv4", false, 0); if (ipAddress.AddressFamily == AddressFamily.InterNetworkV6) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.IpV6, "IPv6", false, 0); throw new ClickHouseException( ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"Parameter \"{ParameterName}\". The type \"{ipAddress.AddressFamily}\" of the network address is not supported."); } return null; } /// /// Checks if the provided string is a valid name for a parameter. /// /// The string to check. /// if the string is a valid parameter name; otherwise . public static bool IsValidParameterName(string? parameterName) { return ValidateParameterName(parameterName, out _); } private static string GetId(string? parameterName) { if (!ValidateParameterName(parameterName, out var id)) throw new ArgumentException("The name of the parameter must be a valid ClickHouse identifier.", nameof(parameterName)); return id; } private static bool ValidateParameterName(string? parameterName, [MaybeNullWhen(false)] out string id) { if (string.IsNullOrWhiteSpace(parameterName)) { id = null; return false; } id = TrimParameterName(parameterName); return ParameterNameRegex.IsMatch(id); } internal ClickHouseParameterMode GetParameterMode(ClickHouseParameterMode inheritParameterMode) { var mode = ParameterMode; if (mode == ClickHouseParameterMode.Inherit) return inheritParameterMode; return mode; } internal static string TrimParameterName(string parameterName) { if (parameterName.Length > 0) { if (parameterName[0] == '{' && parameterName[^1] == '}') return parameterName[1..^1]; // MSSQL-style parameter name if (parameterName[0] == '@') return parameterName[1..]; } return parameterName; } private class ParameterColumnWriterBuilder : ITypeDispatcher { private readonly string _parameterId; [AllowNull] private readonly object _value; private readonly ClickHouseColumnSettings? _columnSettings; private readonly IClickHouseColumnTypeInfo _typeInfo; public ParameterColumnWriterBuilder(string parameterId, object? value, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo typeInfo) { _parameterId = parameterId; _value = value; _columnSettings = columnSettings; _typeInfo = typeInfo ?? throw new ArgumentNullException(nameof(typeInfo)); } public IClickHouseColumnWriter Dispatch() { var singleElementColumn = new ConstantReadOnlyList((T) _value, 1); return _typeInfo.CreateColumnWriter(_parameterId, singleElementColumn, _columnSettings); } } private class ParameterColumnTypeDescriptorAdapter : IClickHouseColumnTypeDescriptor { private readonly ClickHouseParameter _parameter; public ClickHouseDbType? ClickHouseDbType => _parameter._forcedType ?? _parameter.GetValueDependentType()?.DbType; public Type ValueType => _parameter?.Value?.GetType() ?? typeof(DBNull); public bool? IsNullable => _parameter._forcedNullable; public int Size => _parameter._size; public byte? Precision => _parameter._forcedPrecision; public byte? Scale => _parameter._forcedScale; public TimeZoneInfo? TimeZone => _parameter.TimeZone; public int? ArrayRank => _parameter._forcedArrayRank; public ParameterColumnTypeDescriptorAdapter(ClickHouseParameter parameter) { _parameter = parameter; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseParameterCollection.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; namespace Octonica.ClickHouseClient { /// /// Represents a collection of parameters associated with a . This class cannot be inherited. /// public sealed class ClickHouseParameterCollection : DbParameterCollection, IList, IReadOnlyList { private readonly List _parameterNames = new List(); private readonly Dictionary _parameters = new Dictionary(StringComparer.OrdinalIgnoreCase); /// public override int Count => _parameters.Count; /// public override object SyncRoot => ((ICollection) _parameters).SyncRoot; /// public override int Add(object value) { if (value == null) throw new ArgumentNullException(nameof(value)); var parameter = (ClickHouseParameter) value; return Add(parameter); } /// /// Creates, adds to the collection and returns a new parameter with specified name and value. /// /// The name of the parameter. /// The value of the paramter. /// A new added to the collection. public ClickHouseParameter AddWithValue(string parameterName, object? value) { var parameter = new ClickHouseParameter(parameterName) {Value = value}; Add(parameter); return parameter; } /// /// Creates, adds to the collection and returns a new parameter with specified name, value and type. /// /// The name of the parameter. /// The value of the paramter. /// The type of the paramter /// A new added to the collection. public ClickHouseParameter AddWithValue(string parameterName, object? value, DbType dbType) { return AddWithValue(parameterName, value, (ClickHouseDbType)dbType); } /// /// Creates, adds to the collection and returns a new parameter with specified name, value and type. /// /// The name of the parameter. /// The value of the paramter. /// The type of the paramter /// A new added to the collection. public ClickHouseParameter AddWithValue(string parameterName, object? value, ClickHouseDbType dbType) { var parameter = new ClickHouseParameter(parameterName) { Value = value, ClickHouseDbType = dbType }; Add(parameter); return parameter; } void ICollection.Add(ClickHouseParameter item) { Add(item); } /// /// Adds an existing parameter to the collection. /// /// The parameter. /// The zero-based index of the parameter in the collection. public int Add(ClickHouseParameter item) { if (item == null) throw new ArgumentNullException(nameof(item)); if (item.Collection != null) { var errorText = ReferenceEquals(item.Collection, this) ? $"The parameter \"{item.ParameterName}\" already belongs to the collection. It can't be added to the same connection twice." : $"The parameter \"{item.ParameterName}\" already belongs to a collection. It can't be added to different collections."; throw new ArgumentException(errorText, nameof(item)); } if (_parameters.ContainsKey(item.Id)) throw new ArgumentException($"A parameter with the name \"{item.ParameterName}\" already exists in the collection.", nameof(item)); _parameters.Add(item.Id, item); var result = _parameterNames.Count; _parameterNames.Add(item.Id); item.Collection = this; return result; } /// public override void Clear() { foreach (var parameter in _parameters.Values) parameter.Collection = null; _parameters.Clear(); _parameterNames.Clear(); } /// public bool Contains(ClickHouseParameter item) { return item != null && _parameters.TryGetValue(item.Id, out var parameter) && ReferenceEquals(item, parameter); } /// public void CopyTo(ClickHouseParameter[] array, int arrayIndex) { int i = arrayIndex; foreach (var key in _parameterNames) array[i++] = _parameters[key]; } /// public override bool Contains(object value) { if (!(value is ClickHouseParameter parameter)) return false; return Contains(parameter); } /// public override int IndexOf(object value) { if (!(value is ClickHouseParameter parameter)) return -1; return IndexOf(parameter); } /// public override void Insert(int index, object value) { if (value == null) throw new ArgumentNullException(nameof(value)); var parameter = (ClickHouseParameter) value; Insert(index, parameter); } /// public bool Remove(ClickHouseParameter item) { if (item == null) throw new ArgumentNullException(nameof(item)); if (!_parameters.TryGetValue(item.Id, out var existingParameter) || !ReferenceEquals(item, existingParameter)) return false; var comparer = _parameters.Comparer; var name = item.Id; var index = _parameterNames.FindIndex(n => comparer.Equals(n, name)); _parameterNames.RemoveAt(index); var result = _parameters.Remove(name); item.Collection = null; Debug.Assert(result); return true; } /// /// Removes the parameter with the specified name from the collection. /// /// The name of the parameter. /// if the parameter was removed; if a parameter with the specified name is not present in the collection. public bool Remove(string parameterName) { return Remove(parameterName, out _); } /// /// Removes the parameter with the specified name from the collection. /// /// The name of the parameter. /// When this method returns, contains the removed parameter or if a parameter was not removed. /// if the parameter was removed; if a parameter with the specified name is not present in the collection. public bool Remove(string parameterName, [MaybeNullWhen(false)] out ClickHouseParameter parameter) { if (parameterName == null) throw new ArgumentNullException(nameof(parameterName)); var name = ClickHouseParameter.TrimParameterName(parameterName); if (!_parameters.Remove(name, out parameter)) return false; parameter.Collection = null; var comparer = _parameters.Comparer; var index = _parameterNames.FindIndex(n => comparer.Equals(n, name)); _parameterNames.RemoveAt(index); return true; } /// public override void Remove(object value) { if (!(value is ClickHouseParameter parameter)) return; Remove(parameter); } /// public int IndexOf(ClickHouseParameter item) { if (item == null) return -1; if (!_parameters.TryGetValue(item.Id, out var existingParameter) || !ReferenceEquals(item, existingParameter)) return -1; var comparer = _parameters.Comparer; var name = item.Id; var index = _parameterNames.FindIndex(n => comparer.Equals(n, name)); return index; } /// public void Insert(int index, ClickHouseParameter item) { if (item == null) throw new ArgumentNullException(nameof(item)); if (item.Collection != null) { var errorText = ReferenceEquals(item.Collection, this) ? $"The parameter \"{item.ParameterName}\" already belongs to the collection. It can't be added to the same connection twice." : $"The parameter \"{item.ParameterName}\" already belongs to a collection. It can't be added to different collections."; throw new ArgumentException(errorText, nameof(item)); } if (_parameters.ContainsKey(item.Id)) throw new ArgumentException($"A parameter with the name \"{item.ParameterName}\" already exists in the collection.", nameof(item)); _parameterNames.Insert(index, item.Id); _parameters.Add(item.Id, item); item.Collection = this; } /// public override void RemoveAt(int index) { var name = _parameterNames[index]; if (_parameters.Remove(name, out var parameter)) parameter.Collection = null; _parameterNames.RemoveAt(index); } /// public override void RemoveAt(string parameterName) { Remove(parameterName, out _); } /// protected override void SetParameter(int index, DbParameter value) { if (value == null) throw new ArgumentNullException(nameof(value)); var name = _parameterNames[index]; var parameter = (ClickHouseParameter) value; var comparer = _parameters.Comparer; if (comparer.Equals(name, parameter.Id)) { var existingParameter = _parameters[name]; if (!ReferenceEquals(parameter, existingParameter)) { if (parameter.Collection != null) { var errorText = ReferenceEquals(parameter.Collection, this) ? $"The parameter \"{parameter.ParameterName}\" already belongs to the collection. It can't be added to the same connection twice." : $"The parameter \"{parameter.ParameterName}\" already belongs to a collection. It can't be added to different collections."; throw new ArgumentException(errorText, nameof(value)); } _parameters[name] = parameter; existingParameter.Collection = null; parameter.Collection = this; } } else { if (parameter.Collection != null) { var errorText = ReferenceEquals(parameter.Collection, this) ? $"The parameter \"{parameter.ParameterName}\" already belongs to the collection. It can't be added to the same connection twice." : $"The parameter \"{parameter.ParameterName}\" already belongs to a collection. It can't be added to different collections."; throw new ArgumentException(errorText, nameof(value)); } if (_parameters.ContainsKey(parameter.Id)) throw new ArgumentException($"A parameter with the name \"{parameter.ParameterName}\" already exists in the collection.", nameof(value)); if(_parameters.Remove(name, out var existingParameter)) existingParameter.Collection = null; _parameters.Add(parameter.Id, parameter); _parameterNames[index] = parameter.Id; parameter.Collection = this; } } /// protected override void SetParameter(string parameterName, DbParameter value) { if (parameterName == null) throw new ArgumentNullException(nameof(parameterName)); if (value == null) throw new ArgumentNullException(nameof(value)); var name = ClickHouseParameter.TrimParameterName(parameterName); var parameter = (ClickHouseParameter) value; var comparer = _parameters.Comparer; if (_parameters.TryGetValue(name, out var existingParameter)) { if (!ReferenceEquals(parameter, existingParameter)) { if (parameter.Collection != null) { var errorText = ReferenceEquals(parameter.Collection, this) ? $"The parameter \"{parameter.ParameterName}\" already belongs to the collection. It can't be added to the same connection twice." : $"The parameter \"{parameter.ParameterName}\" already belongs to a collection. It can't be added to different collections."; throw new ArgumentException(errorText, nameof(value)); } } if (comparer.Equals(name, parameter.Id)) { _parameters[name] = parameter; } else { if (_parameters.ContainsKey(parameter.Id)) throw new ArgumentException($"A parameter with the name \"{parameter.ParameterName}\" already exists in the collection.", nameof(value)); var index = _parameterNames.FindIndex(n => comparer.Equals(n, name)); _parameterNames[index] = parameter.Id; _parameters.Remove(name); _parameters.Add(parameter.Id, parameter); } if (!ReferenceEquals(parameter, existingParameter)) { existingParameter.Collection = null; parameter.Collection = this; } } else if (comparer.Equals(name, parameter.Id)) { Add(parameter); } else { throw new ArgumentException( $"A parameter with the name \"{parameterName}\" is not present in the collection. It can't be replaced with the parameter \"{parameter.ParameterName}\".", nameof(parameterName)); } } /// public override int IndexOf(string parameterName) { if (parameterName == null) throw new ArgumentNullException(nameof(parameterName)); var name = ClickHouseParameter.TrimParameterName(parameterName); var comparer = _parameters.Comparer; return _parameterNames.FindIndex(n => comparer.Equals(n, name)); } /// public override bool Contains(string value) { if (value == null) throw new ArgumentNullException(nameof(value)); var parameterName = ClickHouseParameter.TrimParameterName(value); return _parameters.ContainsKey(parameterName); } /// public override void CopyTo(Array array, int index) { if (array == null) throw new ArgumentNullException(nameof(array)); var i = index; foreach (var name in _parameterNames) array.SetValue(_parameters[name], i++); } IEnumerator IEnumerable.GetEnumerator() { return _parameterNames.Select(n => _parameters[n]).GetEnumerator(); } /// public override IEnumerator GetEnumerator() { return _parameterNames.Select(n => _parameters[n]).GetEnumerator(); } /// protected override DbParameter GetParameter(int index) { return _parameters[_parameterNames[index]]; } /// protected override DbParameter GetParameter(string parameterName) { if (!TryGetValue(parameterName, out var parameter)) throw new ArgumentException($"Parameter \"{parameterName}\" not found.", nameof(parameterName)); return parameter; } /// public override void AddRange(Array values) { if (values == null) throw new ArgumentNullException(nameof(values)); foreach (var parameter in values.Cast()) Add(parameter); } /// /// Adds the specified parameters to the collection. /// /// The set of parameters that should be added to this collection. /// This operation is not atomic, it calls for each parameter in . public void AddRange(IEnumerable parameters) { if (parameters == null) throw new ArgumentNullException(nameof(parameters)); foreach (var parameter in parameters) Add(parameter); } /// /// Gets the with the specified name. /// /// The name of the parameter. /// /// When this method returns, contains the with the specified name or /// if a parameter is not present in the collection. /// /// if the parameter with the specified name was found in the collection; otherwise public bool TryGetValue(string parameterName, [NotNullWhen(true)] out ClickHouseParameter? parameter) { if (parameterName == null) throw new ArgumentNullException(nameof(parameterName)); var name = ClickHouseParameter.TrimParameterName(parameterName); return _parameters.TryGetValue(name, out parameter); } internal void OnParameterIdChanged(string originalId, ClickHouseParameter parameter) { Debug.Assert(ReferenceEquals(parameter.Collection, this)); if (_parameters.Comparer.Equals(originalId, parameter.Id)) return; SetParameter(originalId, parameter); } /// public new ClickHouseParameter this[int index] { get => (ClickHouseParameter) base[index]; set => base[index] = value; } /// Gets or sets the with the specified name. /// The name of the in the collection. /// The with the specified name. public new ClickHouseParameter this[string parameterName] { get => (ClickHouseParameter) base[parameterName]; set => base[parameterName] = value; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseParameterMode.cs ================================================ #region License Apache 2.0 /* Copyright 2022-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// Specifies the list of available modes of passing parameters to the query. /// public enum ClickHouseParameterMode { /// /// The default mode. Currently the default mode is . /// Default = 0, /// /// This value indicates that the mode should be inherited. /// Parameters inherit the mode from a command. /// A command inherits the mode from a connection. /// For a connection this value is equivalent to . /// Inherit = 1, /// /// This value indicates that parameters should be passed to the query in the binary format. This is the default mode. /// /// /// In this mode parameters will be passed to the query as a table with the single row. Each parameter will be replaced by SELECT subquery. /// This mode doesn't allow to pass parameters in parts of the query where scalar subqueries are not allowed. /// Binary = 2, /// /// This value indicates that parameters should be passed to the query as constant literals. /// /// /// In this mode parameters' values will be interpolated to the query string as constant literals. /// This mode allows to use parameters in any part of the query where a constant is allowed. /// For sending parameters without modifying the query use the mode . /// Interpolate = 3, /// /// This value indicates that parameters should be passed as key-value pairs where values are literals. /// Parameters will not be inlined to the query. /// /// /// Unlike other modes this one relies on the server-side support. An error will be raised if the server doesn't support this mode. /// Serialize = 4, } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHousePasswordComplexityRule.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// Describes a password complexity rule provided by the ClickHouse server. /// public sealed class ClickHousePasswordComplexityRule { /// /// Gets the rule pattern, provided by the server. /// public string OriginalPattern { get; } /// /// Gets the rule message. /// public string ExceptionMessage { get; } /// /// Initializes a new instance of with specified arguments. /// /// The rule pattern. /// Ther rule message. public ClickHousePasswordComplexityRule(string originalPattern, string exceptionMessage) { OriginalPattern = originalPattern; ExceptionMessage = exceptionMessage; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseQueryExecutionProgress.cs ================================================ #region License Apache 2.0 /* Copyright 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// Represents information about a query execution progress. /// public readonly struct ClickHouseQueryExecutionProgress { /// /// The number of rows processed by a query. /// public ulong Rows { get; } /// /// The number of bytes processed by a query. /// public ulong Bytes { get; } /// /// The total number of rows processed by a query. /// public ulong TotalRows { get; } /// /// The total number of rows processed by a query. /// public ulong TotalBytes { get; } /// /// The number of rows written by a query. /// public ulong WrittenRows { get; } /// /// The number of bytes processed by a query. /// public ulong WrittenBytes { get; } /// /// The time elapsed from the start of a query execution in nanoseconds. /// public ulong ElapsedNanoseconds { get; } /// /// Creates an instance of a query execution process. /// public ClickHouseQueryExecutionProgress(ulong rows, ulong bytes, ulong totalRows, ulong totalBytes, ulong writtenRows, ulong writtenBytes, ulong elapsedNanoseconds) { Rows = rows; Bytes = bytes; TotalRows = totalRows; TotalBytes = totalBytes; WrittenRows = writtenRows; WrittenBytes = writtenBytes; ElapsedNanoseconds = elapsedNanoseconds; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseReaderColumnSettings.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Types; using System.Diagnostics; namespace Octonica.ClickHouseClient { internal readonly struct ClickHouseReaderColumnSettings { public ClickHouseColumnSettings? Column { get; } public IClickHouseColumnReinterpreter? Reinterpreter { get; } public ClickHouseReaderColumnSettings(ClickHouseColumnSettings? column, IClickHouseColumnReinterpreter? reinterpreter) { Column = column; Reinterpreter = reinterpreter; } public ClickHouseReaderColumnSettings WithColumnSettings(string columnName, ClickHouseColumnSettings column, IClickHouseColumnReinterpreter? reinterpreter) { if (Reinterpreter != null && Column?.ColumnType == null) { if (reinterpreter != null) { Debug.Assert(column.ColumnType != null); throw new ClickHouseException(ClickHouseErrorCodes.InvalidColumnSettings, $"An external callback function for converting values was defined for the column \"{columnName}\". The type of the column was implicityly defined and can't be redefined in the column settings."); } return new ClickHouseReaderColumnSettings(column, Reinterpreter); } Debug.Assert((column.ColumnType == null) == (reinterpreter == null)); return new ClickHouseReaderColumnSettings(column, reinterpreter); } public ClickHouseReaderColumnSettings WithUserDefinedReader(string columnName, IClickHouseColumnReinterpreter? reinterpreter) { if (reinterpreter == null) { if (Column?.ColumnType != null || Reinterpreter == null) return this; return new ClickHouseReaderColumnSettings(Column, null); } if(Column?.ColumnType != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidColumnSettings, $"An external callback function for converting values can't be set for the column \"{columnName}\" because the type of the column was already defined in the column settings."); return new ClickHouseReaderColumnSettings(Column, reinterpreter); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseServerInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.ObjectModel; namespace Octonica.ClickHouseClient { /// /// Describes a ClickHouse server. This class can't be inherited. /// public sealed class ClickHouseServerInfo { /// /// Gets the name of the server provided by the server when opening a connection. /// public string Name { get; } /// /// Gets the version of the server. /// public ClickHouseVersion Version { get; } /// /// Gets the revision of the ClickHouse binary protocol negotiated between the client and the server. /// public int Revision { get; } /// /// Gets the revision of the ClickHouse binary protocol supported by the server. This value can't be less than the negotiated revision (). /// public int ServerRevision { get; } /// /// Gets the default timezone of the server. /// public string Timezone { get; } /// /// Gets the display name of the server provided by the server when opening a connection. /// public string DisplayName { get; } /// /// Gets password complexity rules provided by the server when opening a connection. /// public ReadOnlyCollection? PasswordComplexityRules { get; } /// /// Initializes a new instance of with specified arguments. /// /// The name of the server provided by the server when opening a connection. /// The version of the server. /// The revision of the ClickHouse binary protocol supported by the server. /// The revision of the ClickHouse binary protocol negotiated between the client and the server. /// The default timezone of the server. /// The display name of the server provided by the server when opening a connection. /// Password complexity rules provided by the server when opening a connection. public ClickHouseServerInfo(string name, ClickHouseVersion version, int serverRevision, int revision, string timezone, string displayName, ReadOnlyCollection? passwordComplexityRules) { Name = name ?? throw new ArgumentNullException(nameof(name)); Version = version; ServerRevision = serverRevision; Revision = revision; Timezone = timezone; DisplayName = displayName; PasswordComplexityRules = passwordComplexityRules; } /// /// Creates a copy of the server info with the specified timezone. /// /// The default timezone of the server. /// A new instance of with the specified timezone. public ClickHouseServerInfo WithTimezone(string timezone) { return new ClickHouseServerInfo( name: Name, version: Version, serverRevision: ServerRevision, revision: Revision, timezone: timezone, displayName: DisplayName, passwordComplexityRules: PasswordComplexityRules); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTable.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.ObjectModel; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient { internal struct ClickHouseTable { public BlockHeader Header { get; } public ReadOnlyCollection Columns { get; } public ClickHouseTable(BlockHeader header, ReadOnlyCollection columns) { Header = header ?? throw new ArgumentNullException(nameof(header)); Columns = columns ?? throw new ArgumentNullException(nameof(columns)); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; namespace Octonica.ClickHouseClient { /// /// Represents a configurable column descriptor. /// public class ClickHouseTableColumn : IClickHouseColumnDescriptor { /// public string ColumnName { get; } /// /// Gets or sets the settings that should be applied when writing the column. /// public ClickHouseColumnSettings? Settings { get; set; } /// /// Gets or sets the type of the column. /// public ClickHouseDbType? ClickHouseDbType { get; set; } /// /// Gets or sets the type of the column's value (i.e. the type of column's cells). /// public Type ValueType { get; } /// /// Gets or sets the value indicating whether the column can contain NULLs. /// public bool? IsNullable { get; set; } /// /// Gets or sets the size. This value is applied to the ClickHouse type FixedString. /// public int Size { get; set; } /// /// Gets or sets the precision. This value is applied to ClickHouse types Decimal and DateTime64. /// public byte? Precision { get; set; } /// /// Gets or sets the scale. This value is applied to the ClickHouse type Decimal. /// public byte? Scale { get; set; } /// /// Gets or sets the time zone. This value is applied to ClickHouse types DateTime and DateTime64. /// public TimeZoneInfo? TimeZone { get; set; } /// /// Gets or sets the rank (a number of dimensions) of an array. /// public int? ArrayRank { get; set; } /// /// Gets the object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// public object Value { get; } /// /// Initializes a new instance of class with the specified name. /// /// The name of the column. /// /// The object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// The type of the column's values. public ClickHouseTableColumn(string columnName, object value, Type valueType) { if (string.IsNullOrWhiteSpace(columnName)) throw new ArgumentException("The name of a column must be a non-empty string", nameof(columnName)); ColumnName = columnName; Value = value ?? throw new ArgumentNullException(nameof(value)); ValueType = valueType ?? throw new ArgumentNullException(nameof(valueType)); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTableColumnCollection.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Collections; using System.Collections.Generic; using System.Globalization; namespace Octonica.ClickHouseClient { /// /// Represents a collection of columns associated with a . This class cannot be inherited. /// public sealed class ClickHouseTableColumnCollection : IndexedCollectionBase { /// /// Initializes a new instance of with the default capacity. /// public ClickHouseTableColumnCollection() : base(StringComparer.OrdinalIgnoreCase) { } /// /// Initializes a new instance of with the specified capacity capacity. /// /// The initial number of elements that the collection can contain. public ClickHouseTableColumnCollection(int capacity) : base(capacity, StringComparer.OrdinalIgnoreCase) { } /// protected sealed override string GetKey(ClickHouseTableColumn item) { return item.ColumnName; } private string GetUniqueColumnName(string baseName) { int i = 0; string name; do { name = string.Format(CultureInfo.InvariantCulture, "{0}{1}", baseName, i++); } while (ContainsKey(name)); return name; } /// /// Creates a new column with the default name and adds it to the collection. /// /// The type of the column's values. /// The list of column's values. /// A new column. public ClickHouseTableColumn AddColumn(IReadOnlyList column) { return AddColumn(GetUniqueColumnName("column"), column); } /// /// Creates a new column with the specified name and adds it to the collection. /// /// The type of the column's values. /// The name of the column. /// The list of column's values. /// A new column. public ClickHouseTableColumn AddColumn(string columnName, IReadOnlyList column) { var result = AddColumn(columnName, column, typeof(T)); if (result.Settings?.ColumnType == null) result.Settings = new ClickHouseColumnSettings(result.Settings?.StringEncoding, result.Settings?.EnumConverter, typeof(T)); return result; } /// /// Creates a new column with the default name and adds it to the collection. /// /// /// The object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// A new column. public ClickHouseTableColumn AddColumn(object column) { return AddColumn(GetUniqueColumnName("column"), column); } /// /// Creates a new column with the specified name and adds it to the collection. /// /// The name of the column. /// /// The object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// A new column. public ClickHouseTableColumn AddColumn(string columnName, object column) { if (column == null) throw new ArgumentNullException(nameof(column)); var columnType = column.GetType(); Type? enumerable = null; Type? altEnumerable = null; Type? asyncEnumerable = null; Type? altAsyncEnumerable = null; Type? readOnlyList = null; Type? altReadOnlyList = null; Type? list = null; Type? altList = null; foreach (var ifs in columnType.GetInterfaces()) { if (!ifs.IsGenericType) continue; var ifsDefinition = ifs.GetGenericTypeDefinition(); if (ifsDefinition == typeof(IEnumerable<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altEnumerable ??= enumerable; enumerable = ifs; } else if (ifsDefinition == typeof(IAsyncEnumerable<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altAsyncEnumerable = asyncEnumerable; asyncEnumerable = ifs; } else if (ifsDefinition == typeof(IReadOnlyList<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altReadOnlyList = readOnlyList; readOnlyList = ifs; } else if (ifsDefinition == typeof(IList<>) && ifs.GetGenericArguments()[0] != typeof(object)) { altList = list; list = ifs; } } Type genericCollectionType; if (readOnlyList != null) { if (altReadOnlyList != null) throw CreateInterfaceAmbiguousException(readOnlyList, altReadOnlyList); genericCollectionType = readOnlyList; } else if (list != null) { if (altList != null) throw CreateInterfaceAmbiguousException(list, altList); genericCollectionType = list; } else if (asyncEnumerable != null) { if (altAsyncEnumerable != null) throw CreateInterfaceAmbiguousException(asyncEnumerable, altAsyncEnumerable); genericCollectionType = asyncEnumerable; } else if (enumerable != null) { if (altEnumerable != null) throw CreateInterfaceAmbiguousException(enumerable, altEnumerable); genericCollectionType = enumerable; } else { throw new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, "The column is not a generic collection. A type of the column can't be detected."); } var elementType = genericCollectionType.GetGenericArguments()[0]; var result = AddColumn(columnName, column, elementType); if (result.Settings?.ColumnType == null) result.Settings = new ClickHouseColumnSettings(result.Settings?.StringEncoding, result.Settings?.EnumConverter, elementType); return result; } /// /// Creates a new column with the default name and adds it to the collection. /// /// /// The object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// The type of the column's values. /// A new column. public ClickHouseTableColumn AddColumn(object column, Type columnType) { return AddColumn(GetUniqueColumnName("column"), column, columnType); } /// /// Creates a new column with the specified name and adds it to the collection. /// /// The name of the column. /// /// The object representing a column. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// The type of the column's values. /// A new column. public ClickHouseTableColumn AddColumn(string columnName, object column, Type columnType) { var result = new ClickHouseTableColumn(columnName, column, columnType); Add(result); return result; } private static ClickHouseException CreateInterfaceAmbiguousException(Type itf, Type altItf) { return new ClickHouseException(ClickHouseErrorCodes.ColumnTypeMismatch, $"A type of the column is ambiguous. The column implements interfaces \"{itf}\" and \"{altItf}\"."); } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTableProvider.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient { /// /// The default implementation of . This class can't be inherited. /// public sealed class ClickHouseTableProvider : IClickHouseTableProvider { /// public string TableName { get; } /// public int ColumnCount => Columns.Count; /// public int RowCount { get; } /// /// Gets the collection of table's columns. /// public ClickHouseTableColumnCollection Columns { get; } = new ClickHouseTableColumnCollection(); /// /// Initializes a new instance of class with specified table name and number of rows. /// /// The name of the table. /// The number of rows in the table. public ClickHouseTableProvider(string tableName, int rowCount) { if (string.IsNullOrWhiteSpace(tableName)) throw new ArgumentNullException("The name of a table must be a non-empty string.", nameof(tableName)); if (rowCount < 0) throw new ArgumentException("The number of rows must be a positive number.", nameof(rowCount)); TableName = tableName; RowCount = rowCount; } /// public ClickHouseTableColumn AddColumn(object column) { return Columns.AddColumn(column); } /// public ClickHouseTableColumn AddColumn(object column, Type columnType) { return Columns.AddColumn(column, columnType); } /// public ClickHouseTableColumn AddColumn(string columnName, object column) { return Columns.AddColumn(columnName, column); } /// public ClickHouseTableColumn AddColumn(string columnName, object column, Type columnType) { return Columns.AddColumn(columnName, column, columnType); } /// public ClickHouseTableColumn AddColumn(IReadOnlyList column) { return Columns.AddColumn(column); } /// public ClickHouseTableColumn AddColumn(string columnName, IReadOnlyList column) { return Columns.AddColumn(columnName, column); } object IClickHouseTableProvider.GetColumn(int index) { return Columns[index].Value; } IClickHouseColumnDescriptor IClickHouseTableProvider.GetColumnDescriptor(int index) { return Columns[index]; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTableProviderCollection.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Utils; using System; namespace Octonica.ClickHouseClient { /// /// Represents a collection of table providers associated with a . This class cannot be inherited. /// public sealed class ClickHouseTableProviderCollection : IndexedCollectionBase { /// /// Initializes a new instance of with the default capacity. /// public ClickHouseTableProviderCollection() : base(StringComparer.OrdinalIgnoreCase) { } /// /// Initializes a new instance of with the specified capacity capacity. /// /// The initial number of elements that the collection can contain. public ClickHouseTableProviderCollection(int capacity) : base(capacity, StringComparer.OrdinalIgnoreCase) { } /// protected sealed override string GetKey(IClickHouseTableProvider item) { return item.TableName; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTableWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Linq; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient { internal sealed class ClickHouseTableWriter : IClickHouseTableWriter { public string TableName { get; } public int RowCount { get; } public IReadOnlyList Columns { get; } public ClickHouseTableWriter(string tableName, int rowCount, IEnumerable columns) { if (columns == null) throw new ArgumentNullException(nameof(columns)); TableName = tableName ?? throw new ArgumentNullException(nameof(tableName)); RowCount = rowCount; Columns = columns.ToList().AsReadOnly(); } public ClickHouseTableWriter(string tableName, int rowCount, IReadOnlyList columns) { TableName = tableName; RowCount = rowCount; Columns = columns; } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTcpClient.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Net; using System.Net.Security; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient { internal sealed class ClickHouseTcpClient : IDisposable { private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1); private readonly TcpClient _client; private readonly ClickHouseConnectionSettings _settings; private IClickHouseTypeInfoProvider _typeInfoProvider; private readonly ClickHouseBinaryProtocolReader _reader; private readonly ClickHouseBinaryProtocolWriter _writer; private readonly SslStream? _sslStream; private Exception? _unhandledException; private int _state; public ClickHouseTcpClientState State => (ClickHouseTcpClientState)_state; public ClickHouseServerInfo ServerInfo { get; private set; } public ClickHouseTcpClient( TcpClient client, ClickHouseBinaryProtocolReader reader, ClickHouseBinaryProtocolWriter writer, ClickHouseConnectionSettings settings, ClickHouseServerInfo serverInfo, IClickHouseTypeInfoProvider typeInfoProvider, SslStream? sslStream) { _reader = reader ?? throw new ArgumentNullException(nameof(reader)); _writer = writer ?? throw new ArgumentNullException(nameof(writer)); _client = client ?? throw new ArgumentNullException(nameof(client)); _settings = settings ?? throw new ArgumentNullException(nameof(settings)); ServerInfo = serverInfo; _typeInfoProvider = typeInfoProvider; _sslStream = sslStream; } public async ValueTask OpenSession(bool async, IClickHouseSessionExternalResources? externalResources, CancellationToken sessionCancellationToken, CancellationToken cancellationToken) { var state = (ClickHouseTcpClientState)_state; if (state != ClickHouseTcpClientState.Failed) { try { if (async) await _semaphore.WaitAsync(cancellationToken); else _semaphore.Wait(cancellationToken); var previousState = (ClickHouseTcpClientState)Interlocked.CompareExchange(ref _state, (int)ClickHouseTcpClientState.Active, (int)ClickHouseTcpClientState.Ready); Debug.Assert(previousState != ClickHouseTcpClientState.Active); state = previousState == ClickHouseTcpClientState.Ready ? ClickHouseTcpClientState.Active : previousState; } catch (ObjectDisposedException) { // Reading an actual state without modification of field _state state = (ClickHouseTcpClientState)Interlocked.CompareExchange(ref _state, (int)state, (int)state); if (state != ClickHouseTcpClientState.Failed) throw; } } if (state == ClickHouseTcpClientState.Failed) { if (_unhandledException != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "Connection is broken.", _unhandledException); throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "Connection is broken."); } try { return new Session(this, externalResources, sessionCancellationToken); } catch { _semaphore.Release(); throw; } } private bool TryInterceptMessage(IServerMessage message) { switch (message.MessageCode) { case ServerMessageCode.TimezoneUpdate: var timezone = ((ServerTimeZoneUpdateMessage)message).Timezone; if (string.Equals(timezone, ServerInfo.Timezone, StringComparison.Ordinal)) return true; var updServerInfo = ServerInfo.WithTimezone(timezone); _typeInfoProvider = _typeInfoProvider.Configure(updServerInfo); ServerInfo = updServerInfo; return true; default: return false; } } private void SetFailed(Exception? unhandledException) { Dispose(false); _unhandledException = unhandledException; // 'Failed' is the terminal state. Plain assignment should work just as well as interlocked operations. _state = (int)ClickHouseTcpClientState.Failed; } public void Dispose() { Dispose(true); } private void Dispose(bool disposing) { _semaphore.Dispose(); _reader.Dispose(); _writer.Dispose(); _sslStream?.Dispose(); // The disposed TcpClient returns null for Client _client.Client?.Close(disposing ? 1 : 0); _client.Dispose(); } public sealed class Session : IDisposable, IAsyncDisposable { private readonly ClickHouseTcpClient _client; private readonly IClickHouseSessionExternalResources? _externalResources; private readonly CancellationToken _sessionCancellationToken; public IClickHouseTypeInfoProvider TypeInfoProvider => _client._typeInfoProvider; public ClickHouseServerInfo ServerInfo => _client.ServerInfo; public bool IsDisposed { get; private set; } public bool IsFailed => _client.State == ClickHouseTcpClientState.Failed; public Session(ClickHouseTcpClient client, IClickHouseSessionExternalResources? externalResources, CancellationToken sessionCancellationToken) { _client = client; _externalResources = externalResources; _sessionCancellationToken = sessionCancellationToken; } public async ValueTask SendQuery( ClientQueryMessage.Builder messageBuilder, IReadOnlyCollection? tables, bool async, CancellationToken cancellationToken) { CheckDisposed(); var writer = _client._writer; ClientQueryMessage queryMessage; try { var settings = _client._settings; messageBuilder.ClientName = settings.ClientName; messageBuilder.ClientVersion = settings.ClientVersion; messageBuilder.Host = settings.Host; messageBuilder.RemoteAddress = ((IPEndPoint?) _client._client.Client.RemoteEndPoint)?.ToString(); messageBuilder.ProtocolRevision = _client.ServerInfo.Revision; messageBuilder.CompressionEnabled = _client._settings.Compress; queryMessage = messageBuilder.Build(); if (queryMessage.Settings != null) { if (_client.ServerInfo.Revision < ClickHouseProtocolRevisions.MinRevisionWithSettingsSerializedAsStrings) { throw new ClickHouseException( ClickHouseErrorCodes.ProtocolRevisionNotSupported, $"Query settings are not supported. Current protocol revision is {_client.ServerInfo.Revision}. Minimal required protocol revision is {ClickHouseProtocolRevisions.MinRevisionWithSettingsSerializedAsStrings}."); } } queryMessage.Write(writer); if (tables != null) { foreach (var table in tables) WriteTable(table); } WriteTable(ClickHouseEmptyTableWriter.Instance); } catch (Exception ex) { writer.Discard(); throw ClickHouseHandledException.Wrap(ex); } await WithCancellationToken(cancellationToken, ct => writer.Flush(async, ct)); return queryMessage; } public async ValueTask SendQuery(ClientQueryMessage queryMessage, bool async, CancellationToken cancellationToken) { CheckDisposed(); var writer = _client._writer; try { queryMessage.Write(writer); WriteTable(ClickHouseEmptyTableWriter.Instance); } catch (Exception ex) { writer.Discard(); throw ClickHouseHandledException.Wrap(ex); } await WithCancellationToken(cancellationToken, ct => writer.Flush(async, ct)); } public async ValueTask SendCancel(bool async) { CheckDisposed(); var writer = _client._writer; writer.Write7BitInt32((int) ClientMessageCode.Cancel); await writer.Flush(async, CancellationToken.None); } public async ValueTask SendPing(bool async, CancellationToken cancellationToken) { CheckDisposed(); var writer = _client._writer; writer.Write7BitInt32((int) ClientMessageCode.Ping); await WithCancellationToken(cancellationToken, ct => writer.Flush(async, ct)); } public async ValueTask SendTable(IClickHouseTableWriter table, bool async, CancellationToken cancellationToken) { CheckDisposed(); try { WriteTable(table); } catch (Exception ex) { _client._writer.Discard(); throw ClickHouseHandledException.Wrap(ex); } await WithCancellationToken(cancellationToken, ct => _client._writer.Flush(async, ct)); } private void WriteTable(IClickHouseTableWriter table) { var writer = _client._writer; writer.Write7BitInt32((int) ClientMessageCode.Data); writer.WriteString(table.TableName); var compression = _client._settings.Compress ? CompressionAlgorithm.Lz4 : CompressionAlgorithm.None; writer.BeginCompress(compression, _client._settings.CompressionBlockSize); writer.WriteByte(BlockFieldCodes.IsOverflows); writer.WriteBool(false); // is overflow writer.WriteByte(BlockFieldCodes.BucketNum); writer.WriteInt32(-1); // data size in block. -1 for null writer.WriteByte(BlockFieldCodes.End); writer.Write7BitInt32(table.Columns.Count); writer.Write7BitInt32(table.RowCount); foreach (var column in table.Columns) { writer.WriteString(column.ColumnName); writer.WriteString(column.ColumnType); if (_client.ServerInfo.Revision >= ClickHouseProtocolRevisions.MinRevisionWithCustomSerialization) writer.WriteBool(false); // has_custom if (table.RowCount == 0) continue; while (true) { var size = writer.WriteRaw(mem => column.WritePrefix(mem.Span)); if (size.Elements == 1) break; if (size.Elements != 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. A column writer returned an unexpected number of prefixes: {size.Elements}."); } int rowCount = table.RowCount; while (rowCount > 0) { var size = writer.WriteRaw(mem => column.WriteNext(mem.Span)); rowCount -= size.Elements; } } writer.EndCompress(); } public async ValueTask ReadMessage(bool async, CancellationToken cancellationToken) { return await WithCancellationToken(cancellationToken, ct => ReadMessageInternal(async, ct)); } private async ValueTask ReadMessageInternal(bool async, CancellationToken cancellationToken) { IServerMessage message; do { message = await _client._reader.ReadMessage(_client.ServerInfo.Revision, true, async, cancellationToken); } while (_client.TryInterceptMessage(message)); return message; } public async ValueTask ReadTable(ServerDataMessage dataMessage, IReadOnlyList? columnSettings, bool async, CancellationToken cancellationToken) { var withDecompression = dataMessage.MessageCode != ServerMessageCode.ProfileEvents; var result = await WithCancellationToken( cancellationToken, ct => ReadTable( withDecompression, (typeInfo, rowCount, mode) => typeInfo.CreateColumnReader(rowCount, mode), (columnInfo, reader, index) => ReadTableColumn(columnInfo, reader, columnSettings == null || columnSettings.Count <= index ? default : columnSettings[index]), async, ct) ); Debug.Assert(result.columns != null); var blockHeader = new BlockHeader(dataMessage.TempTableName, result.columnInfos.AsReadOnly(), result.rowCount); return new ClickHouseTable(blockHeader, result.columns.AsReadOnly()); } public async ValueTask SkipTable(ServerDataMessage dataMessage, bool async, CancellationToken cancellationToken) { var withDecompression = dataMessage.MessageCode != ServerMessageCode.ProfileEvents; var result = await WithCancellationToken(cancellationToken, ct => ReadTable(withDecompression, (typeInfo, rowCount, mode) => typeInfo.CreateSkippingColumnReader(rowCount, mode), null, async, ct)); return new BlockHeader(dataMessage.TempTableName, result.columnInfos.AsReadOnly(), result.rowCount); } private async ValueTask<(List columnInfos, List? columns, int rowCount)> ReadTable( bool withDecompression, Func createColumnReader, Func? readTableColumn, bool async, CancellationToken cancellationToken) where TReader : IClickHouseColumnReaderBase { CheckDisposed(); var compression = withDecompression && _client._settings.Compress ? CompressionAlgorithm.Lz4 : CompressionAlgorithm.None; var reader = _client._reader; reader.BeginDecompress(compression); int blockFieldCode; bool isOverflows = false; // It seems that this value is used only for internal purposes and does not affect the format of the block int bucketNum = -1; do { blockFieldCode = await reader.Read7BitInt32(async, cancellationToken); switch (blockFieldCode) { case BlockFieldCodes.IsOverflows: isOverflows = await reader.ReadBool(async, cancellationToken); break; case BlockFieldCodes.BucketNum: bucketNum = await reader.ReadInt32(async, cancellationToken); break; case BlockFieldCodes.End: break; default: throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Internal error. Unexpected block field code (0x{blockFieldCode:X}) received from the server."); } } while (blockFieldCode != BlockFieldCodes.End); var columnCount = await reader.Read7BitInt32(async, cancellationToken); var rowCount = await reader.Read7BitInt32(async, cancellationToken); if (isOverflows) throw new NotImplementedException("TODO: implement support for is_overflows."); var columnInfos = new List(columnCount); var columns = readTableColumn == null ? null : new List(columnCount); for (int i = 0; i < columnCount; i++) { var columnName = await reader.ReadString(async, cancellationToken); var columnTypeName = await reader.ReadString(async, cancellationToken); var serializationMode = ClickHouseColumnSerializationMode.Default; if (_client.ServerInfo.Revision >= ClickHouseProtocolRevisions.MinRevisionWithCustomSerialization) { var hasCustom = await reader.ReadBool(async, cancellationToken); if (hasCustom) serializationMode = ClickHouseColumnSerializationMode.Custom; } var columnType = _client._typeInfoProvider.GetTypeInfo(columnTypeName); var columnInfo = new ColumnInfo(columnName, columnType); columnInfos.Add(columnInfo); var columnReader = createColumnReader(columnType, rowCount, serializationMode); if (rowCount > 0) { while (true) { var sequenceSize = await reader.ReadRaw(columnReader.ReadPrefix, async, cancellationToken); if (sequenceSize.Elements == 1) break; if (sequenceSize.Elements != 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Received an unexpected number of column prefixes: {sequenceSize.Elements}."); } } var columnRowCount = rowCount; while (columnRowCount > 0) { var sequenceSize = await reader.ReadRaw(columnReader.ReadNext, async, cancellationToken); if (sequenceSize.Elements < 0) throw new InvalidOperationException("The number of elements must be greater than zero."); if (sequenceSize.Elements > columnRowCount) throw new InvalidOperationException($"The number of rows in the column \"{columnName}\" is greater than the number of rows in the table."); if (sequenceSize.Elements < columnRowCount) await reader.Advance(async, cancellationToken); columnRowCount -= sequenceSize.Elements; } if (columns != null) { Debug.Assert(readTableColumn != null); var column = readTableColumn(columnInfo, columnReader, i); columns.Add(column); } } reader.EndDecompress(); return (columnInfos, columns, rowCount); } private static IClickHouseTableColumn ReadTableColumn(ColumnInfo columnInfo, IClickHouseColumnReader columnReader, ClickHouseReaderColumnSettings settings) { var column = columnReader.EndRead(settings.Column); var reinterpreter = settings.Reinterpreter; if (reinterpreter != null) { var reinterpretedColumn = reinterpreter.TryReinterpret(column); if (reinterpretedColumn != null) return reinterpretedColumn; return new ReinterpretedTableColumn(column, CreateCastFunc(columnInfo, reinterpreter.BuiltInConvertToType, reinterpreter.ExternalConvertToType)); } return column; } private static Func CreateCastFunc(ColumnInfo columnInfo, Type? builtInConvertToType, Type? externalConvertToType) { if ((externalConvertToType ?? builtInConvertToType) == typeof(object)) return value => value; // This is fine, everything is an object return CastFailed; object CastFailed(object value) { if (value == DBNull.Value) return value; if (builtInConvertToType != null) { if (externalConvertToType == null) { throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"A value from the column \"{columnInfo.Name}\" of type \"{columnInfo.TypeInfo.GetFieldType()}\" can't be converted to type \"{builtInConvertToType}\". " + "This type is defined in the column settings."); } throw new ClickHouseException( ClickHouseErrorCodes.ColumnTypeMismatch, $"A value from the column \"{columnInfo.Name}\" of type \"{columnInfo.TypeInfo.GetFieldType()}\" can't be converted to type \"{builtInConvertToType}\". " + $"This type is implicitly defined by the callback function for a value conversion. The callback function returns a value of type \"{externalConvertToType}\"."); } throw new ClickHouseException( ClickHouseErrorCodes.InternalError, $"ClickHouseClient didn't find a suitable type conversion for the column \"{columnInfo.Name}\" of type \"{columnInfo.TypeInfo.GetFieldType()}\". " + "This is an internal error. Please, report a bug if you see this message."); } } private async ValueTask WithCancellationToken(CancellationToken token, Func> execute) { if (_sessionCancellationToken == CancellationToken.None) return await execute(token); if (token == CancellationToken.None) return await execute(_sessionCancellationToken); using var linkedTs = CancellationTokenSource.CreateLinkedTokenSource(token, _sessionCancellationToken); try { return await execute(linkedTs.Token); } catch (TaskCanceledException taskCanceledEx) { if (token.IsCancellationRequested) throw new TaskCanceledException(taskCanceledEx.Message, taskCanceledEx, token); throw; } catch (OperationCanceledException operationCanceledEx) { if (token.IsCancellationRequested) throw new OperationCanceledException(operationCanceledEx.Message, operationCanceledEx, token); throw; } } private async ValueTask WithCancellationToken(CancellationToken token, Func execute) { if (_sessionCancellationToken == CancellationToken.None) { await execute(token); return; } if (token == CancellationToken.None) { await execute(_sessionCancellationToken); return; } using var linkedTs = CancellationTokenSource.CreateLinkedTokenSource(token, _sessionCancellationToken); try { await execute(linkedTs.Token); } catch (TaskCanceledException taskCanceledEx) { if (token.IsCancellationRequested) throw new TaskCanceledException(taskCanceledEx.Message, taskCanceledEx, token); throw; } catch (OperationCanceledException operationCanceledEx) { if (token.IsCancellationRequested) throw new OperationCanceledException(operationCanceledEx.Message, operationCanceledEx, token); throw; } } private void CheckDisposed() { if (IsDisposed) throw new ObjectDisposedException("Internal error. This object was disposed and no more has an exclusive access to the network stream."); if (_client.State == ClickHouseTcpClientState.Failed) { if (_client._unhandledException != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "Connection is broken.", _client._unhandledException); throw new ClickHouseException(ClickHouseErrorCodes.InvalidConnectionState, "Connection is broken."); } } public async ValueTask SetFailed(Exception? unhandledException, bool sendCancel, bool async) { if (IsDisposed) return null; Exception? networkException = null; if (sendCancel) { try { await SendCancel(async); } catch (Exception ex) { networkException = new ClickHouseException(ClickHouseErrorCodes.NetworkError, "Network error. Operation was not canceled properly.", ex); } } _client.SetFailed(unhandledException); Exception? externalException = null; if (_externalResources != null) { externalException = await _externalResources.ReleaseOnFailure(unhandledException, async); if (ReferenceEquals(unhandledException, externalException)) externalException = null; } var exceptions = new List(3); if (unhandledException != null) exceptions.Add(unhandledException); if (networkException != null) exceptions.Add(networkException); if (externalException != null) exceptions.Add(externalException); switch (exceptions.Count) { case 0: return null; case 1: return exceptions[0]; default: return new AggregateException(exceptions); } } public void Dispose() { TaskHelper.WaitNonAsyncTask(Dispose(false)); } public ValueTask DisposeAsync() { return Dispose(true); } public ValueTask Dispose(bool async) { if (IsDisposed || IsFailed) return default; Interlocked.CompareExchange(ref _client._state, (int)ClickHouseTcpClientState.Ready, (int)ClickHouseTcpClientState.Active); _client._semaphore.Release(); IsDisposed = true; return _externalResources?.Release(async) ?? default; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTcpClientState.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { internal enum ClickHouseTcpClientState { /// /// TCP client is ready to open a session /// Ready = 0, /// /// There is an active session associated with the client /// Active = 1, /// /// A session was failed. TCP client was forced to close /// Failed = 2 } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseTlsMode.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// Specifies the list of available options for establishing a secure connection with the TLS protocol. /// public enum ClickHouseTlsMode { /// /// TLS is disabled. Data exchange between the client and the server will be performed without encryption. /// The connection will fail to open if the server requires TLS. /// Disable = 0, /// /// TLS is required. The connection will fail to open if the server doesn't support TLS. /// Require = 1 } } ================================================ FILE: src/Octonica.ClickHouseClient/ClickHouseVersion.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Globalization; namespace Octonica.ClickHouseClient { /// /// Represents a version of the ClickHouse client or of the ClickHouse server. /// public readonly struct ClickHouseVersion : IEquatable { /// /// Gets the value of the major component of the version. /// public int Major { get; } /// /// Gets the value of the minor component of the version. /// public int Minor { get; } /// /// Gets the value of the build component of the version. /// public int Build { get; } /// /// Initializes a new instance of with the specified components. /// /// The value of the major component of the version. /// the value of the minor component of the version. /// the value of the build component of the version. public ClickHouseVersion(int major, int minor, int build) { Major = major; Minor = minor; Build = build; } /// /// Returns the string representation of the version in the format {Major}.{Minor}.{Build}. /// /// The string representation of the version. public override string ToString() { return string.Format(CultureInfo.InvariantCulture, "{0}.{1}.{2}", Major, Minor, Build); } /// public bool Equals(ClickHouseVersion other) { return Major == other.Major && Minor == other.Minor && Build == other.Build; } /// public override bool Equals(object? obj) { return obj is ClickHouseVersion other && Equals(other); } /// public override int GetHashCode() { unchecked { int hashCode = Major; hashCode = (hashCode * 397) ^ Minor; hashCode = (hashCode * 397) ^ Build; return hashCode; } } /// /// Parses the string in the format "{Major}.{Minor}.{Build}". /// /// The string to parse. /// The parsed . /// Throws an when the value is null. /// Throws an when the value is not a valid string representation of the . public static ClickHouseVersion Parse(string value) { if (value == null) throw new ArgumentNullException(nameof(value)); var firstIndex = value.IndexOf('.'); var lastIndex = firstIndex < 0 ? -1 : value.IndexOf('.', firstIndex + 1); var nextIndex = lastIndex < 0 ? -1 : value.IndexOf('.', lastIndex + 1); const NumberStyles numberStyle = NumberStyles.Integer & ~NumberStyles.AllowLeadingSign; if (firstIndex < 0) { if (int.TryParse(value, numberStyle, CultureInfo.InvariantCulture, out var major)) return new ClickHouseVersion(major, 0, 0); } else if (nextIndex < 0) { var span = value.AsSpan(); if (int.TryParse(span.Slice(0, firstIndex), numberStyle, CultureInfo.InvariantCulture, out var major)) { if (lastIndex < 0) { if (int.TryParse(span.Slice(firstIndex + 1), numberStyle, CultureInfo.InvariantCulture, out var minor)) return new ClickHouseVersion(major, minor, 0); } else { if (int.TryParse(span.Slice(firstIndex + 1, lastIndex - firstIndex - 1), numberStyle, CultureInfo.InvariantCulture, out var minor) && int.TryParse(span.Slice(lastIndex + 1), numberStyle, CultureInfo.InvariantCulture, out var build)) { return new ClickHouseVersion(major, minor, build); } } } } throw new ArgumentException("The value is not a valid version number.", nameof(value)); } /// /// Compares two objects of type . /// /// The left operand. /// The right operand. /// if the two objects are equal; oterwise . public static bool operator ==(ClickHouseVersion left, ClickHouseVersion right) { return left.Equals(right); } /// /// Compares two objects of type . /// /// The left operand. /// The right operand. /// if the two objects are equal; oterwise . public static bool operator !=(ClickHouseVersion left, ClickHouseVersion right) { return !left.Equals(right); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Exceptions/ClickHouseErrorCodes.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Exceptions { /// /// The static class that provides access to error codes. /// public static class ClickHouseErrorCodes { /// /// The code for an unspecified error. This code means that an error of an unknown origin was wrapped in . /// public const int Unspecified = 0; /// /// The code for an error provided by the ClickHouse server. /// public const int ServerError = 1; /// /// The code for an error caused by an invalid state of the connection. /// public const int InvalidConnectionState = 2; /// /// The code for an error caused by an attempt to execute an operation when the connection is closed. /// public const int ConnectionClosed = 3; /// /// The code for an error caused by an unexpected response from the server. /// public const int ProtocolUnexpectedResponse = 4; /// /// The code for an error caused by an attempt to use a feature that is not supported by the revision negotiated between the client and the server. /// public const int ProtocolRevisionNotSupported = 5; /// /// The code for an error caused by an attempt to read or write a column of an unknown type. /// public const int TypeNotSupported = 6; /// /// The code for an error caused by an invalid full name of the type. This code usually means that the type was specified with invalid parameters. /// public const int InvalidTypeName = 7; /// /// The code for an error caused by an attempt to use a parametrizable type without parameters. /// public const int TypeNotFullySpecified = 8; /// /// The code for an error caused by a lack of an expected response from the server. /// public const int QueryTypeMismatch = 9; /// /// The code for a data reading error. /// public const int DataReaderError = 10; /// /// The code for an error caused by a lack of required query parameter. /// public const int QueryParameterNotFound = 11; /// /// The code for an error caused by a configuration of the query parameter (). /// public const int InvalidQueryParameterConfiguration = 12; /// /// An obsolete error code. It was replaced with , , and . /// [Obsolete("ColumnMismatch was replaced with " + nameof(ColumnTypeMismatch) + ", " + nameof(NotSupportedInSyncronousMode) + " and " + nameof(InvalidRowCount))] public const int ColumnMismatch = 13; /// /// The common code for unexpected situations. This code indicates errors (bugs) in the client's code. /// public const int InternalError = 14; /// /// The code for an error caused by a compression decoder. /// public const int CompressionDecoderError = 15; /// /// The code for an error caused by a network error. /// public const int NetworkError = 16; /// /// The code for an error caused by invalid settings (). /// public const int InvalidColumnSettings = 17; /// /// The code for a column type error. This code means that there is no mapping between the ClickHouse column's type and the provided type. /// public const int ColumnTypeMismatch = 18; /// /// The code for an error caused by a wrong number of rows in a column. Usually this error code means that there are less rows in the colum than expected. /// public const int InvalidRowCount = 19; /// /// The code for an error caused by an attempt to execute an asyncronous operation in a syncronous method. /// public const int NotSupportedInSyncronousMode = 20; /// /// The code for an error caused by the callback to an external code. /// public const int CallbackError = 21; /// /// The code for an error caused by a violation of the TLS protocol. /// public const int TlsError = 22; /// /// The code for an error caused by unexpected changes in the table's structure. /// public const int TableModified = 23; } } ================================================ FILE: src/Octonica.ClickHouseClient/Exceptions/ClickHouseException.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Exceptions { /// /// Represents an exception specific to ClickHouse client or server. /// public class ClickHouseException : Exception { /// /// The numeric code of the error. For the full list of errors see . /// public int ErrorCode { get; } /// /// Initializes a new instance of the exception with the specified error code. /// /// The code of the error. It should be one of the values from . public ClickHouseException(int errorCode) { ErrorCode = errorCode; } /// /// Initializes a new instance of the exception with specified error code and message. /// /// The code of the error. It should be one of the values from . /// The error message that explains the reason for the exception. public ClickHouseException(int errorCode, string? message) : base(message) { ErrorCode = errorCode; } /// /// Initializes a new instance of the exception with specified error code, message, and inner exception. /// /// The code of the error. It should be one of the values from . /// The error message that explains the reason for the exception. /// The exception that is the cause of the current exception. public ClickHouseException(int errorCode, string? message, Exception? innerException) : base(message, innerException) { ErrorCode = errorCode; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Exceptions/ClickHouseHandledException.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Exceptions { /// /// An exception which doesn't break the connection. It always has . This class can't be inherited. /// public sealed class ClickHouseHandledException : ClickHouseException { private ClickHouseHandledException(int errorCode, string message, Exception innerException) : base(errorCode, message, innerException) { } internal static ClickHouseHandledException Wrap(Exception exception) { if (exception is ClickHouseHandledException nfException) return nfException; if (exception is ClickHouseException chException) return new ClickHouseHandledException(chException.ErrorCode, chException.Message, chException); return new ClickHouseHandledException(ClickHouseErrorCodes.Unspecified, exception.Message, exception); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Exceptions/ClickHouseServerException.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Exceptions { /// /// Represents an exception generated by the ClickHouse server. The of an instance of this class /// is always . This class can not be inherited. /// public sealed class ClickHouseServerException : ClickHouseException { /// /// The code of the error provided by the server. /// public int ServerErrorCode { get; } /// /// The type of the error provided by the server. /// public string ServerErrorType { get; } /// /// The server-side stack trace. /// public string ServerStackTrace { get; } /// /// The query passed to the server or if the error is not related to the query. /// public string? Query { get; } /// /// Initializes a new instance of the ClickHouse server exception with the provided parameters and the /// equal to . /// /// The code of the error provided by the server. /// The type of the error provided by the server. /// The error message that explains the reason for the exception. /// The server-side stack trace. public ClickHouseServerException(int serverErrorCode, string serverErrorType, string? message, string serverStackTrace) : base(ClickHouseErrorCodes.ServerError, message) { ServerErrorCode = serverErrorCode; ServerErrorType = serverErrorType; ServerStackTrace = serverStackTrace; } /// /// Initializes a new instance of the ClickHouse server exception with the provided parameters and the /// equal to . /// /// The code of the error provided by the server. /// The type of the error provided by the server. /// The error message that explains the reason for the exception. /// The server-side stack trace. /// The exception that is the cause of the current exception. public ClickHouseServerException(int serverErrorCode, string serverErrorType, string? message, string serverStackTrace, Exception? innerException) : base(ClickHouseErrorCodes.ServerError, message, innerException) { ServerErrorCode = serverErrorCode; ServerErrorType = serverErrorType; ServerStackTrace = serverStackTrace; } private ClickHouseServerException(int serverErrorCode, string serverErrorType, string? message, string serverStackTrace, string? query) : base(ClickHouseErrorCodes.ServerError, message) { ServerErrorCode = serverErrorCode; ServerErrorType = serverErrorType; ServerStackTrace = serverStackTrace; Query = query; } private ClickHouseServerException(int serverErrorCode, string serverErrorType, string? message, string serverStackTrace, string? query, Exception? innerException) : base(ClickHouseErrorCodes.ServerError, message, innerException) { ServerErrorCode = serverErrorCode; ServerErrorType = serverErrorType; ServerStackTrace = serverStackTrace; Query = query; } /// /// Creates and returns a new instance of with the provided query. /// /// The query that should be added to the exception. /// The new instance of with the provided query public ClickHouseServerException CopyWithQuery(string query) { if (InnerException == null) return new ClickHouseServerException(ServerErrorCode, ServerErrorType, Message, ServerStackTrace, query); return new ClickHouseServerException(ServerErrorCode, ServerErrorType, Message, ServerStackTrace, query, InnerException); } } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient { /// /// The interface for an object representing a column with arrays. /// /// The type of the array's element. public interface IClickHouseArrayTableColumn : IClickHouseTableColumn { /// /// Copies elements from the array to the specified buffer. /// /// The zero-based index of the row. /// The buffer into which to copy data. /// The index within the row from which to begin the copy operation. /// The actual number of copied elements. int CopyTo(int index, Span buffer, int dataOffset); } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseColumnDescriptor.cs ================================================ #region License Apache 2.0 /* Copyright 2021-2022 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Types; namespace Octonica.ClickHouseClient { /// /// Represents a set of properties describing a column and its type. /// public interface IClickHouseColumnDescriptor : IClickHouseColumnTypeDescriptor { /// /// Gets the name of the column. /// string ColumnName { get; } /// /// Gets the settings that should be applied when writing the column. /// ClickHouseColumnSettings? Settings { get; } } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseSessionExternalResources.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Threading.Tasks; namespace Octonica.ClickHouseClient { internal interface IClickHouseSessionExternalResources { ValueTask Release(bool async); ValueTask ReleaseOnFailure(Exception? exception, bool async); } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient { /// /// The basic interface for 's internal columns. /// public interface IClickHouseTableColumn { /// /// Gets the number of rows in the column. /// int RowCount { get; } /// /// Gets the value indicating whether an actual value at the specified index is NULL. /// /// The zero-based index of row. /// if an actual value at the specified index is NULL; otherwise . bool IsNull(int index); /// /// Gets the value at the specified index. /// /// The zero-based index of row. /// The value at the specified index or if the value is NULL. object GetValue(int index); /// /// Makes an attempt to convert this column to a column with the values of the type . /// /// The desired type of column's values. /// The column converted to the type or if such conversion is not supported. /// This method may or may not return when the column itself implements the interface . IClickHouseTableColumn? TryReinterpret(); /// /// Makes an attempt to convert this column to an array column with the type of array's element . /// /// The desired type of array elements. /// The column converted to the type or if such conversion is not supported. /// This method may or may not return when the column itself implements the interface . IClickHouseArrayTableColumn? TryReinterpretAsArray() => null; /// /// If possible, performs double dispatch and provides this object as an instance of . /// /// The type of the dispatched value. /// The dispatcher that receives an instance of . /// When this method returns, the result of the dispatch operation, if the dispatcher was called; otherwise the default value of . /// if this object is an instance of and the dispatch operation was performed; otherwise . bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue); } /// /// The generic interface for 's internal columns. /// public interface IClickHouseTableColumn : IClickHouseTableColumn { /// /// Gets the default value of the column for the sparse serialization. /// T DefaultValue { get; } /// /// Gets the value at the specified index. /// /// The zero-based index of row. /// The value at the specified index. /// This method should never return . new T GetValue(int index); } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseTableColumnDispatcher.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient { /// /// The interface for objects performing an arbitrary operation over the generic column. /// /// The type of a value returned by the dispatcher. public interface IClickHouseTableColumnDispatcher { /// /// When implemented in a derived class performs an arbitrary operation over the column. /// /// The type of the column's values. /// The column. /// The result of the operation. TRes Dispatch(IClickHouseTableColumn column); } } ================================================ FILE: src/Octonica.ClickHouseClient/IClickHouseTableProvider.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections; using System.Collections.Generic; namespace Octonica.ClickHouseClient { /// /// The basic interface for an object that provides access to a table along with metadata. /// public interface IClickHouseTableProvider { /// /// Gets the name of the table. /// public string TableName { get; } /// /// Gets the number of columns in the table. /// public int ColumnCount { get; } /// /// Gets the number of rows in the table. /// public int RowCount { get; } /// /// Gets the descriptor of the column at the specified index. /// /// The zero-based index of the column. /// The descriptor of the column. IClickHouseColumnDescriptor GetColumnDescriptor(int index); /// /// Returns the object that represents the column at the specified index. It must implement one of interfaces: /// , /// , /// , /// or /// . /// /// The zero-based index of the column. /// The object that represents the column at the specified index. object GetColumn(int index); } } ================================================ FILE: src/Octonica.ClickHouseClient/Octonica.ClickHouseClient.csproj ================================================  netcoreapp3.1;net6.0;net8.0 enable true true $(ClickHouseClientVersion) 2.2.10 $(Version).0 $(Version)$(ClickHouseClientVersionSuffix) Octonica © 2019 – 2026 Octonica Octonica.ClickHouseClient https://github.com/Octonica/ClickHouseClient ClickHouse .NET Core driver Apache-2.0 Octonica ClickHouse https://github.com/Octonica/ClickHouseClient.git git ================================================ FILE: src/Octonica.ClickHouseClient/Properties/AssemblyInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Octonica.ClickHouseClient.Tests")] ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/BlockFieldCodes.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { /// /// https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/BlockInfo.h /// internal static class BlockFieldCodes { public const int End = 0; /// /// * After running GROUP BY ... WITH TOTALS with the max_rows_to_group_by and group_by_overflow_mode = 'any' settings, /// * a row is inserted in the separate block with aggregated values that have not passed max_rows_to_group_by. /// * If it is such a block, then is_overflows is set to true for it. /// public const int IsOverflows = 1; /// ///* When using the two-level aggregation method, data with different key groups are scattered across different buckets. ///* In this case, the bucket number is indicated here. It is used to optimize the merge for distributed aggregation. ///* Otherwise -1. /// public const int BucketNum = 2; } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/BlockHeader.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.ObjectModel; namespace Octonica.ClickHouseClient.Protocol { internal class BlockHeader { private static readonly ReadOnlyCollection EmptyColumns = new ReadOnlyCollection(new ColumnInfo[0]); public string? TableName { get; } public ReadOnlyCollection Columns { get; } public int RowCount { get; } public BlockHeader(string? tableName, ReadOnlyCollection? columns, int rowCount) { TableName = tableName; Columns = columns ?? EmptyColumns; RowCount = rowCount; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/CityHash.cs ================================================ #region License // Copyright (c) 2011 Google, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. // // CityHash, by Geoff Pike and Jyrki Alakuijala // // This file provides CityHash128() and related functions. // // It's probably possible to create even faster hash functions by // writing a program that systematically explores some of the space of // possible hash functions, by using SIMD instructions, or by // compromising on hash quality. #endregion using System; using System.Buffers; using System.Runtime.CompilerServices; namespace Octonica.ClickHouseClient.Protocol { // CityHash v1.0.2 implementation from ClickHouse internal static class CityHash { private static UInt64 UNALIGNED_LOAD64(ReadOnlySequence p) { if (p.FirstSpan.Length > sizeof(UInt64)) return BitConverter.ToUInt64(p.FirstSpan); Span tmpBuffer = stackalloc byte[sizeof(UInt64)]; p.Slice(0, sizeof(UInt64)).CopyTo(tmpBuffer); return BitConverter.ToUInt64(tmpBuffer); } private static UInt32 UNALIGNED_LOAD32(ReadOnlySequence p) { if (p.FirstSpan.Length > sizeof(UInt32)) return BitConverter.ToUInt32(p.FirstSpan); Span tmpBuffer = stackalloc byte[sizeof(UInt32)]; p.Slice(0, sizeof(UInt32)).CopyTo(tmpBuffer); return BitConverter.ToUInt32(tmpBuffer); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt64 Fetch64(ReadOnlySequence p) { return UNALIGNED_LOAD64(p); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt32 Fetch32(ReadOnlySequence p) { return UNALIGNED_LOAD32(p); } // Some primes between 2^63 and 2^64 for various uses. private const UInt64 k0 = 0xc3a5c85c97cb3127UL; private const UInt64 k1 = 0xb492b66fbe98f273UL; private const UInt64 k2 = 0x9ae16a3b2f90404fUL; private const UInt64 k3 = 0xc949d7c7509e6557UL; // Bitwise right rotate. Normally this will compile to a single // instruction, especially if the shift is a manifest constant. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt64 Rotate(UInt64 val, int shift) { // Avoid shifting by 64: doing so yields an undefined result. return shift == 0 ? val : ((val >> shift) | (val << (64 - shift))); } // Equivalent to Rotate(), but requires the second arg to be non-zero. // On x86-64, and probably others, it's possible for this to compile // to a single instruction if both args are already in registers. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt64 RotateByAtLeast1(UInt64 val, int shift) { return (val >> shift) | (val << (64 - shift)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt64 ShiftMix(UInt64 val) { return val ^ (val >> 47); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static UInt64 HashLen16(UInt64 u, UInt64 v) { return Hash128to64(new UInt128(v, u)); } // Hash 128 input bits down to 64 bits of output. // This is intended to be a reasonably good hash function. private static UInt64 Hash128to64(UInt128 x) { // Murmur-inspired hashing. const UInt64 kMul = 0x9ddfea08eb382d69UL; unchecked { #if NET8_0_OR_GREATER var low = (UInt64)x; var high = (UInt64)(x >> 64); UInt64 a = (low ^ high) * kMul; a ^= (a >> 47); UInt64 b = (high ^ a) * kMul; #else UInt64 a = (x.Low ^ x.High) * kMul; a ^= (a >> 47); UInt64 b = (x.High ^ a) * kMul; #endif b ^= (b >> 47); b *= kMul; return b; } } private static UInt64 HashLen0to16(ReadOnlySequence s) { var len = (ulong) s.Length; unchecked { if (len > 8) { UInt64 a = Fetch64(s); UInt64 b = Fetch64(s.Slice((int) len - 8)); return HashLen16(a, RotateByAtLeast1(b + len, (int) len)) ^ b; } if (len >= 4) { UInt64 a = Fetch32(s); return HashLen16(len + (a << 3), Fetch32(s.Slice((int) len - 4))); } if (len > 0) { byte a = s.FirstSpan[0]; byte b = s.Slice((int) len >> 1).FirstSpan[0]; byte c = s.Slice((int) len - 1).FirstSpan[0]; UInt32 y = a + ((uint) b << 8); UInt32 z = (uint) (len + ((uint) c << 2)); return ShiftMix(y * k2 ^ z * k3) * k2; } return k2; } } // Return a 16-byte hash for 48 bytes. Quick and dirty. // Callers do best to use "random-looking" values for a and b. private static (UInt64 first, UInt64 second) WeakHashLen32WithSeeds( UInt64 w, UInt64 x, UInt64 y, UInt64 z, UInt64 a, UInt64 b) { a += w; b = Rotate(b + a + z, 21); UInt64 c = a; a += x; a += y; b += Rotate(a, 44); return (a + z, b + c); } // Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. private static (UInt64 first, UInt64 second) WeakHashLen32WithSeeds( ReadOnlySequence s, UInt64 a, UInt64 b) { return WeakHashLen32WithSeeds( Fetch64(s), Fetch64(s.Slice(8)), Fetch64(s.Slice(16)), Fetch64(s.Slice(24)), a, b); } // A subroutine for CityHash128(). Returns a decent 128-bit hash for strings // of any length representable in ssize_t. Based on City and Murmur. private static UInt128 CityMurmur(ReadOnlySequence s, UInt128 seed) { var len = (ulong) s.Length; unchecked { #if NET8_0_OR_GREATER UInt64 a = (UInt64)seed; UInt64 b = (UInt64)(seed >> 64); #else UInt64 a = seed.Low; UInt64 b = seed.High; #endif UInt64 c = 0; UInt64 d = 0; int l = (int) len - 16; if (l <= 0) { // len <= 16 a = ShiftMix(a * k1) * k1; c = b * k1 + HashLen0to16(s); d = ShiftMix(a + (len >= 8 ? Fetch64(s) : c)); } else { // len > 16 c = HashLen16(Fetch64(s.Slice((int)len - 8)) + k1, a); d = HashLen16(b + len, c + Fetch64(s.Slice((int) len - 16))); a += d; do { a ^= ShiftMix(Fetch64(s) * k1) * k1; a *= k1; b ^= a; c ^= ShiftMix(Fetch64(s.Slice(8)) * k1) * k1; c *= k1; d ^= c; s = s.Slice(16); l -= 16; } while (l > 0); } a = HashLen16(a, c); b = HashLen16(d, b); return new UInt128(HashLen16(b, a), a ^ b); } } public static UInt128 CityHash128WithSeed(ReadOnlySequence s, UInt128 seed) { var len = (ulong) s.Length; if (len < 128) { return CityMurmur(s, seed); } // We expect len >= 128 to be the common case. Keep 56 bytes of state: // v, w, x, y, and z. unchecked { (UInt64 first, UInt64 second) v, w; #if NET8_0_OR_GREATER UInt64 x = (UInt64)seed; UInt64 y = (UInt64)(seed>>64); #else UInt64 x = seed.Low; UInt64 y = seed.High; #endif UInt64 z = len * k1; v.first = Rotate(y ^ k1, 49) * k1 + Fetch64(s); v.second = Rotate(v.first, 42) * k1 + Fetch64(s.Slice(8)); w.first = Rotate(y + z, 35) * k1 + x; w.second = Rotate(x + Fetch64(s.Slice(88)), 53) * k1; // This is the same inner loop as CityHash64(), manually unrolled. var sOrig = s; do { x = Rotate(x + y + v.first + Fetch64(s.Slice(16)), 37) * k1; y = Rotate(y + v.second + Fetch64(s.Slice(48)), 42) * k1; x ^= w.second; y ^= v.first; z = Rotate(z ^ w.first, 33); v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); w = WeakHashLen32WithSeeds(s.Slice(32), z + w.second, y); var tmp = x; x = z; z = tmp; s = s.Slice(64); x = Rotate(x + y + v.first + Fetch64(s.Slice(16)), 37) * k1; y = Rotate(y + v.second + Fetch64(s.Slice(48)), 42) * k1; x ^= w.second; y ^= v.first; z = Rotate(z ^ w.first, 33); v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); w = WeakHashLen32WithSeeds(s.Slice(32), z + w.second, y); tmp = x; x = z; z = tmp; s = s.Slice(64); len -= 128; } while (len >= 128); var offset = sOrig.Length - s.Length; y += Rotate(w.first, 37) * k0 + z; x += Rotate(v.first + z, 49) * k0; // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s. for (int tail_done = 0; tail_done < (int) len;) { tail_done += 32; y = Rotate(y - x, 42) * k0 + v.second; w.first += Fetch64(sOrig.Slice((int) len - tail_done + 16 + offset)); x = Rotate(x, 49) * k0 + w.first; w.first += v.first; v = WeakHashLen32WithSeeds(sOrig.Slice((int) len - tail_done + offset), v.first, v.second); } // At this point our 48 bytes of state should contain more than // enough information for a strong 128-bit hash. We use two // different 48-byte-to-8-byte hashes to get a 16-byte final result. x = HashLen16(x, v.first); y = HashLen16(y, w.first); return new UInt128( HashLen16(x + w.second, y + v.second), HashLen16(x + v.second, w.second) + y); } } public static UInt128 CityHash128(ReadOnlySequence s) { var len = (ulong) s.Length; if (len >= 16) { return CityHash128WithSeed( s.Slice(16), new UInt128( Fetch64(s.Slice(8)), Fetch64(s) ^ k3)); } else if (len >= 8) { return CityHash128WithSeed( ReadOnlySequence.Empty, new UInt128( Fetch64(s.Slice((int)len - 8)) ^ k1, Fetch64(s) ^ unchecked(len * k0))); } else { return CityHash128WithSeed(s, new UInt128(k1, k0)); } } } #if !NET8_0_OR_GREATER internal readonly struct UInt128 { public ulong Low { get; } public ulong High { get; } public UInt128(ulong high, ulong low) { Low = low; High = high; } } #endif } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClickHouseEmptyTableWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; namespace Octonica.ClickHouseClient.Protocol { internal class ClickHouseEmptyTableWriter : IClickHouseTableWriter { public static readonly ClickHouseEmptyTableWriter Instance = new ClickHouseEmptyTableWriter(); public string TableName { get; } public int RowCount => 0; public IReadOnlyList Columns { get; } private ClickHouseEmptyTableWriter() { TableName = string.Empty; Columns = new IClickHouseColumnWriter[0]; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClickHouseParameterWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; using System; using System.Text; namespace Octonica.ClickHouseClient.Protocol { internal abstract class ClickHouseParameterWriter : IClickHouseParameterValueWriter { public abstract int Length { get; } public abstract int Write(Memory buffer); public abstract StringBuilder Interpolate(StringBuilder queryBuilder); public abstract StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeProvider, Func writeValue); public static ClickHouseParameterWriter Dispatch(IClickHouseColumnTypeInfo typeInfo, object? value) { var dispatcher = new Dispatcher(typeInfo, value); return dispatcher.Dispatch(); } private sealed class Dispatcher : ITypeDispatcher { private readonly IClickHouseColumnTypeInfo _typeInfo; private readonly object _value; public Dispatcher(IClickHouseColumnTypeInfo typeInfo, object? value) { _typeInfo = typeInfo; _value = value ?? DBNull.Value; } public ClickHouseParameterWriter Dispatch() { var value = (T)_value; var writer = _typeInfo.CreateParameterWriter(); if (!writer.TryCreateParameterValueWriter(value, isNested: false, out var valueWriter)) valueWriter = null; return new ClickHouseParameterWriter(writer, value, valueWriter); } public ClickHouseParameterWriter Dispatch() { return TypeDispatcher.Dispatch(_value.GetType(), this); } } } internal sealed class ClickHouseParameterWriter : ClickHouseParameterWriter { private readonly IClickHouseParameterWriter _writer; private readonly T _value; private readonly IClickHouseParameterValueWriter? _valueWriter; public override int Length => _valueWriter?.Length ?? 0; public ClickHouseParameterWriter(IClickHouseParameterWriter writer, T value, IClickHouseParameterValueWriter? valueWriter) { _writer = writer; _value = value; _valueWriter = valueWriter; } public override StringBuilder Interpolate(StringBuilder queryBuilder) { return _writer.Interpolate(queryBuilder, _value); } public override StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeProvider, Func writerValue) { if (_valueWriter == null) { // Here we do not have a real value writer, so we can't pass the parameter to the query. We have to interpolate the value directly into the query. return Interpolate(queryBuilder); } return _writer.Interpolate(queryBuilder, typeProvider, (qb, t, w) => w(qb, b => writerValue(b, t))); } public override int Write(Memory buffer) { return _valueWriter?.Write(buffer) ?? 0; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClickHouseProtocolRevisions.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { /// /// The static class that provides access to the ClickHouse's binary protocol revision numbers. /// public static class ClickHouseProtocolRevisions { /// /// The number of the current revision. It is the latest revision supported by the client. /// public const int CurrentRevision = MinRevisionWithSshAuthentication; /// /// The number of protocol's revision that supports SSH authentication. /// public const int MinRevisionWithSshAuthentication = 54466; /// /// The number of protocol's revision with the timzone in data messages. /// public const int MinRevisionWithTimezoneUpdates = 54464; /// /// The number of protocol's revision with the number of total bytes in progress messages. /// public const int MinRevisionWithTotalBytesInProgress = 54463; /// /// The number of the protocol's revision that supports interserver secret (V2). /// public const int MinRevisionWithInterserverSecretV2 = 54462; /// /// The number of protocol's revision with the support of password complexity rules. /// public const int MinRevisionWithPasswordComplexityRules = 54461; /// /// The number of protocol's revision with the number of elapsed nanoseconds in progress messages. /// public const int MinRevisionWithServerQueryTimeInProgress = 54460; /// /// The number of protocol's revision with the support of parameters passed along with the query. /// public const int MinRevisionWithParameters = 54459; /// /// The number of protocol's revision with support of hello message addendum and quota keys. /// public const int MinRevisionWithAddendum = 54458; /// /// The number of protocol's revision with support of custom serialization. /// public const int MinRevisionWithCustomSerialization = 54454; /// /// The number of protocol's revision with support for parallel reading from replicas. /// public const int MinRevisionWithParallelReplicas = 54453; /// /// The number of protocol's revision with the initial query start time. /// public const int MinRevisionWithInitialQueryStartTime = 54449; /// /// The number of protocol's revision that supports distributed depth. /// public const int MinRevisionWithDistributedDepth = 54448; /// /// The number of protocol's revision with the support of Open Telemetry headers. /// public const int MinRevisionWithOpenTelemetry = 54442; /// /// The number of the protocol's revision that supports interserver secret. /// public const int MinRevisionWithInterserverSecret = 54441; /// /// The number of the protocol's revision with settings serialized as strings. /// public const int MinRevisionWithSettingsSerializedAsStrings = 54429; /// /// The minimal number of the revision supported by the client. /// public const int MinSupportedRevision = 54423; } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClickHouseSyntaxHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.RegularExpressions; namespace Octonica.ClickHouseClient.Protocol { internal static class ClickHouseSyntaxHelper { private static readonly Regex IdentifierRegex = new Regex("^[a-zA-Z_][0-9a-zA-Z_]*$"); private const string EscapeChars = "''\"\"\\\\0\0a\ab\be\u001bf\fn\nr\rt\tv\v"; private static bool TryGetUnescapedCharacter(char ch, out char unescapedCh) { for (int i = 0; i < EscapeChars.Length; i += 2) { if (ch == EscapeChars[i]) { unescapedCh = EscapeChars[i + 1]; return true; } } unescapedCh = default; return false; } public static int GetIdentifierLiteralLength(string str, int startIndex) { return GetIdentifierLiteralLength(((ReadOnlySpan)str).Slice(startIndex)); } public static int GetIdentifierLiteralLength(ReadOnlySpan str) { if (str.IsEmpty) return -1; if (str[0] == '`') { var length = GetQuotedTokenLength(str, '`'); if (length > 2) return length; } else if ((str[0] >= 'a' && str[0] <= 'z') || (str[0] >= 'A' && str[0] <= 'Z') || str[0] == '_') { int length = 1; for (; length < str.Length; length++) { var ch = str[length]; if (char.IsWhiteSpace(ch)) break; if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_') continue; return -1; } return length; } return -1; } public static int GetSingleQuoteStringLength(string str, int startIndex) { return GetSingleQuoteStringLength(((ReadOnlySpan)str).Slice(startIndex)); } public static int GetSingleQuoteStringLength(ReadOnlySpan str) { return GetQuotedTokenLength(str, '\''); } public static int GetQuotedTokenLength(ReadOnlySpan str, char quoteSign) { if (str.IsEmpty || str[0] != quoteSign) return -1; int idx = 1; while (idx < str.Length) { var slice = str.Slice(idx); var nextIdx = slice.IndexOfAny(quoteSign, '\\'); if (nextIdx < 0) return -1; if (slice[nextIdx] == '\\') { idx += nextIdx + 2; continue; } return idx + nextIdx + 1; } return -1; } public static string GetIdentifier(ReadOnlySpan identifierLiteral) { if (identifierLiteral.Length == 0) throw new ArgumentException($"The string \"{identifierLiteral.ToString()}\" is not a valid identifier.", nameof(identifierLiteral)); var sb = new StringBuilder(identifierLiteral.Length); if (identifierLiteral[0] == '`') { if (!TryParseQuotedToken(identifierLiteral, '`', out var parsedLiteral) || parsedLiteral == string.Empty) throw new ArgumentException($"The string \"{identifierLiteral.ToString()}\" is not a valid identifier.", nameof(identifierLiteral)); return parsedLiteral; } if ((identifierLiteral[0] >= 'a' && identifierLiteral[0] <= 'z') || (identifierLiteral[0] >= 'A' && identifierLiteral[0] <= 'Z') || identifierLiteral[0] == '_') { sb.Append(identifierLiteral[0]); for (int i = 1; i < identifierLiteral.Length; i++) { var ch = identifierLiteral[i]; if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_') { sb.Append(ch); continue; } throw new ArgumentException($"The string \"{identifierLiteral.ToString()}\" is not a valid identifier.", nameof(identifierLiteral)); } return sb.ToString(); } throw new ArgumentException($"The string \"{identifierLiteral.ToString()}\" is not a valid identifier.", nameof(identifierLiteral)); } public static string GetSingleQuoteString(ReadOnlySpan stringToken) { if (!TryParseQuotedToken(stringToken, '\'', out var result)) throw new ArgumentException($"The value \"{stringToken.ToString()}\" is not a valid string token."); return result; } private static bool TryParseQuotedToken(ReadOnlySpan token, char quoteSign, [MaybeNullWhen(false)] out string value) { if (token.Length < 2 || token[0] != quoteSign || token[^1] != quoteSign) { value = null; return false; } var sb = new StringBuilder(token.Length); for (int i = 1; i < token.Length - 1; i++) { if (token[i] == '\\') { if (i + 1 == token.Length - 1) { value = null; return false; } if (token[i + 1] == quoteSign) { sb.Append(quoteSign); } else if (TryGetUnescapedCharacter(token[i + 1], out var unescapedCh)) { sb.Append(unescapedCh); } else { sb.Append(token[i]).Append(token[i + 1]); } i++; } else { sb.Append(token[i]); } } value = sb.ToString(); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClientHelloMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Protocol { internal class ClientHelloMessage : IClientMessage { public ClientMessageCode MessageCode => ClientMessageCode.Hello; public string ClientName { get; } public ClickHouseVersion ClientVersion { get; } public int ProtocolRevision { get; } public string? Database { get; } public string User { get; } public string? Password { get; } public string? QuotaKey { get; } private ClientHelloMessage(Builder builder) { ClientName = builder.ClientName ?? throw new ArgumentException("The name of the client can't be null.", nameof(ClientName)); ClientVersion = builder.ClientVersion ?? throw new ArgumentException("The version of the client can't be null.", nameof(ClientVersion)); ProtocolRevision = builder.ProtocolRevision ?? throw new ArgumentException("The revision of the protocol is required.", nameof(ProtocolRevision)); Database = builder.Database; User = builder.User ?? throw new ArgumentException("The name of the user is required.", nameof(User)); Password = builder.Password; QuotaKey = builder.QuotaKey; } public void Write(ClickHouseBinaryProtocolWriter writer) { writer.Write7BitInt32((int) MessageCode); writer.WriteString(ClientName); writer.Write7BitInt32(ClientVersion.Major); writer.Write7BitInt32(ClientVersion.Minor); writer.Write7BitInt32(ProtocolRevision); writer.WriteString(Database ?? string.Empty); writer.WriteString(User); writer.WriteString(Password ?? string.Empty); } public void WriteAddendum(ClickHouseBinaryProtocolWriter writer) { writer.WriteString(QuotaKey ?? string.Empty); } internal class Builder { /// /// Required /// public string? ClientName { get; set; } /// /// Required /// public ClickHouseVersion? ClientVersion { get; set; } /// /// Required /// public int? ProtocolRevision { get; set; } /// /// Required /// public string? Database { get; set; } /// /// Required /// public string? User { get; set; } /// /// Optional /// public string? Password { get; set; } /// /// Optional (addendum) /// public string? QuotaKey { get; set; } public ClientHelloMessage Build() { return new ClientHelloMessage(this); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClientMessageCode.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal enum ClientMessageCode { Hello = 0, Query = 1, Data = 2, Cancel = 3, Ping = 4, } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ClientQueryMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023, 2025 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ClientQueryMessage : IClientMessage { public ClientMessageCode MessageCode => ClientMessageCode.Query; public QueryKind QueryKind { get; } public string? QueryId { get; } public string RemoteAddress { get; } public string Host { get; } public string ClientName { get; } public ClickHouseVersion ClientVersion { get; } public int ProtocolRevision { get; } public string Query { get; } public bool CompressionEnabled { get; } // https://github.com/ClickHouse/ClickHouse/blob/master/dbms/src/Core/Settings.h public IReadOnlyCollection>? Settings { get; } public IReadOnlyCollection>? Parameters { get; } private ClientQueryMessage(Builder builder) { QueryKind = builder.QueryKind ?? throw new ArgumentException("The kind of the query is required.", nameof(QueryKind)); QueryId = string.IsNullOrEmpty(builder.QueryId) ? null : builder.QueryId; RemoteAddress = builder.RemoteAddress ?? throw new ArgumentException("The remote address is required.", nameof(RemoteAddress)); Host = builder.Host ?? throw new ArgumentException("The name of the host is required.", nameof(Host)); ClientName = builder.ClientName ?? throw new ArgumentException("The name of the client is required.", nameof(ClientName)); ClientVersion = builder.ClientVersion ?? throw new ArgumentException("The version of the client is required.", nameof(ClientVersion)); ProtocolRevision = builder.ProtocolRevision ?? throw new ArgumentException("The revision of the protocol is required.", nameof(ProtocolRevision)); Query = builder.Query ?? throw new ArgumentException("The query is required.", nameof(Query)); CompressionEnabled = builder.CompressionEnabled ?? throw new ArgumentException("Unknown compression mode.", nameof(CompressionEnabled)); Settings = builder.Settings == null || builder.Settings.Count == 0 ? null : builder.Settings; Parameters = builder.Parameters == null || builder.Parameters.Count == 0 ? null : builder.Parameters; } public void Write(ClickHouseBinaryProtocolWriter writer) { writer.Write7BitInt32((int) MessageCode); writer.WriteString(QueryId ?? string.Empty); switch (QueryKind) { case QueryKind.NoQuery: break; case QueryKind.InitialQuery: writer.Write7BitInt32((int) QueryKind); writer.WriteString(string.Empty); //initial user writer.WriteString(string.Empty); //initial query id writer.WriteString(RemoteAddress); //initial IP address if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithInitialQueryStartTime) { // Initial query start time in microseconds. An actual value of this property should be set by the server. Span zero = stackalloc byte[sizeof(ulong)]; writer.WriteBytes(zero); } writer.Write7BitInt32(1); //TCP writer.WriteString(string.Empty); //OS user writer.WriteString(Host); writer.WriteString(ClientName); writer.Write7BitInt32(ClientVersion.Major); writer.Write7BitInt32(ClientVersion.Minor); writer.Write7BitInt32(ProtocolRevision); writer.WriteString(string.Empty); //quota key if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithDistributedDepth) writer.Write7BitInt32(0); //distributed depth writer.Write7BitInt32(ClientVersion.Build); if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithOpenTelemetry) writer.WriteByte(0); // TODO: add support for Open Telemetry headers if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithParallelReplicas) { writer.WriteByte(0); // collaborate_with_initiator writer.WriteByte(0); // count_participating_replicas writer.WriteByte(0); // number_of_current_replica } break; case QueryKind.SecondaryQuery: throw new NotImplementedException(); default: throw new NotSupportedException(); } if (Settings != null) { // All settings are serialized as strings. Before each value the flag `is_important` is serialized. // https://github.com/ClickHouse/ClickHouse/blob/97d97f6b2e50ab3cf21a25a18cbf1aa327f242e5/src/Core/BaseSettings.h#L19 const int isImportantFlag = 0x1; foreach (var pair in Settings) { writer.WriteString(pair.Key); writer.Write7BitInt32(isImportantFlag); writer.WriteString(pair.Value); } } writer.WriteString(string.Empty); // empty string is a marker of the end of the settings if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithInterserverSecret) writer.WriteString(string.Empty); writer.Write7BitInt32(StateCodes.Complete); writer.WriteBool(CompressionEnabled); writer.WriteString(Query); if (ProtocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithParameters) { if (Parameters != null) { const int isCustomFlag = 0x2; foreach (var pair in Parameters) { if (pair.Value.Length < 0) continue; writer.WriteString(pair.Key); writer.Write7BitInt32(isCustomFlag); var lenght = pair.Value.Length; writer.Write7BitInt32(lenght + 2); writer.WriteByte((byte)'\''); if (lenght > 0) { var size = writer.WriteRaw(lenght, buffer => new SequenceSize(pair.Value.Write(buffer), 1)); // The lenght must be calculated by the parameter writer correctly if (size.Bytes != lenght) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. The length of the parameter \"{pair.Key}\" in bytes is {lenght}, but the number of written bytes is {size.Bytes}."); } writer.WriteByte((byte)'\''); } } writer.WriteString(string.Empty); // empty string is a marker of the end of parameters } else if (Parameters != null) { var errMsg = "The server doesn't support parameters in the query. " + $"This error is caused by one or more parameters passed in the mode \"{nameof(ClickHouseParameterMode.Serialize)}\". " + $"Only \"{nameof(ClickHouseParameterMode.Binary)}\" or \"{nameof(ClickHouseParameterMode.Interpolate)}\" modes are supported."; throw new ClickHouseException(ClickHouseErrorCodes.ProtocolRevisionNotSupported, errMsg); } } public class Builder { /// /// Required /// public QueryKind? QueryKind { get; set; } /// /// Optional /// public string? QueryId { get; set; } /// /// Required /// public string? RemoteAddress { get; set; } /// /// Required /// public string? Host { get; set; } /// /// Required /// public string? ClientName { get; set; } /// /// Required /// public ClickHouseVersion? ClientVersion { get; set; } /// /// Required /// public int? ProtocolRevision { get; set; } /// /// Required /// public string? Query { get; set; } /// /// Required /// public bool? CompressionEnabled { get; set; } /// /// Optional /// public IReadOnlyCollection>? Settings { get; set; } /// /// Optional /// public IReadOnlyCollection>? Parameters { get; set; } public ClientQueryMessage Build() { return new ClientQueryMessage(this); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ColumnInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Types; namespace Octonica.ClickHouseClient.Protocol { internal class ColumnInfo { public string Name { get; } public IClickHouseColumnTypeInfo TypeInfo { get; } public ColumnInfo(string name, IClickHouseColumnTypeInfo typeInfo) { Name = name; TypeInfo = typeInfo; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/CompressionAlgorithm.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal enum CompressionAlgorithm { None = 0, Lz4 = 1 } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/CompressionDecoderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Protocol { internal abstract class CompressionDecoderBase : IDisposable { private readonly int _bufferSize; private byte[]? _compressedBuffer; private byte[]? _decompressedBuffer; private int _compressedPosition; private int _compressedSize; private int _decompressedAvailable; private int _decompressedSize; protected abstract byte AlgorithmIdentifier { get; } public abstract CompressionAlgorithm Algorithm { get; } public bool IsCompleted => _compressedPosition == _compressedSize; protected CompressionDecoderBase(int bufferSize) { _bufferSize = bufferSize; } public int ReadHeader(ReadOnlySequence sequence) { if (!IsCompleted) throw new ClickHouseException(ClickHouseErrorCodes.CompressionDecoderError, "Can't start reading of a new block because reading of the current compressed block is not finished."); const int cityHashSize = 16, headerSize = sizeof(byte) + 2 * sizeof(int); if (sequence.Length < cityHashSize + headerSize) return -1; byte algorithmIdentifier = sequence.Slice(cityHashSize).FirstSpan[0]; if (algorithmIdentifier != AlgorithmIdentifier) { throw new ClickHouseException( ClickHouseErrorCodes.CompressionDecoderError, $"An unexpected compression algorithm identifier was received. Expected value: 0x{AlgorithmIdentifier:X}. Actual value: 0x{algorithmIdentifier:X}."); } var slice = sequence.Slice(cityHashSize + sizeof(byte)); Span intBuffer = stackalloc byte[sizeof(int)]; slice.Slice(0, sizeof(int)).CopyTo(intBuffer); _compressedSize = BitConverter.ToInt32(intBuffer); _compressedSize -= headerSize; slice.Slice(sizeof(int), sizeof(int)).CopyTo(intBuffer); var decompressedSize = BitConverter.ToInt32(intBuffer); _compressedPosition = 0; if (_compressedBuffer == null || _compressedBuffer.Length < _compressedSize) _compressedBuffer = new byte[GetBufferSize(_compressedSize)]; if (_decompressedBuffer == null) { Debug.Assert(_decompressedAvailable == 0); _decompressedBuffer = new byte[GetBufferSize(decompressedSize)]; } else if (_decompressedBuffer.Length < decompressedSize + _decompressedAvailable) { var newBuffer = new byte[GetBufferSize(decompressedSize + _decompressedAvailable)]; if (_decompressedAvailable > 0) Array.Copy(_decompressedBuffer, _decompressedSize - _decompressedAvailable, newBuffer, 0, _decompressedAvailable); _decompressedBuffer = newBuffer; } else { for (int i = 0, j = _decompressedSize - _decompressedAvailable; j < _decompressedSize; i++, j++) _decompressedBuffer[i] = _decompressedBuffer[j]; } _decompressedSize = decompressedSize + _decompressedAvailable; return cityHashSize + headerSize; } public ReadOnlySequence Read() { if (!IsCompleted || _decompressedBuffer == null || _compressedBuffer == null) return ReadOnlySequence.Empty; return new ReadOnlySequence(_decompressedBuffer, _decompressedSize - _decompressedAvailable, _decompressedAvailable); } public void AdvanceReader(SequencePosition position) { if (!ReferenceEquals(position.GetObject(), _decompressedBuffer)) throw new ArgumentException("The position doesn't belong to the sequence.", nameof(position)); var arrayIndex = position.GetInteger(); if (arrayIndex < 0) arrayIndex = unchecked(arrayIndex - int.MinValue); var relativePosition = arrayIndex - (_decompressedSize - _decompressedAvailable); if (relativePosition < 0) throw new ArgumentOutOfRangeException(nameof(position), "The position must be a non-negative number."); if (relativePosition == 0) return; if (relativePosition > _decompressedAvailable) throw new ArgumentOutOfRangeException(nameof(position), "The position must not be greater then the length."); _decompressedAvailable -= relativePosition; } public int ConsumeNext(ReadOnlySequence sequence) { if (_compressedPosition == _compressedSize || _compressedBuffer == null) return 0; var sequencePart = sequence; if (sequence.Length > _compressedSize - _compressedPosition) sequencePart = sequence.Slice(0, _compressedSize - _compressedPosition); sequencePart.CopyTo(((Span) _compressedBuffer).Slice(_compressedPosition)); _compressedPosition += (int) sequencePart.Length; if (_compressedPosition == _compressedSize) { var sourceSpan = new Span(_compressedBuffer, 0, _compressedSize); var targetSpan = new Span(_decompressedBuffer, _decompressedAvailable, _decompressedSize - _decompressedAvailable); _decompressedAvailable += Decode(sourceSpan, targetSpan); Debug.Assert(_decompressedAvailable == _decompressedSize); } return (int) sequencePart.Length; } public void Reset() { _compressedPosition = 0; _compressedSize = 0; _decompressedAvailable = 0; _decompressedSize = 0; } protected abstract int Decode(ReadOnlySpan source, Span target); public abstract void Dispose(); private int GetBufferSize(int minRequiredSize) { int size = _bufferSize; while (size < minRequiredSize) size *= 2; return size; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/CompressionEncoderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Utils; #if NET8_0_OR_GREATER using System.Runtime.InteropServices; #endif namespace Octonica.ClickHouseClient.Protocol { internal abstract class CompressionEncoderBase: IDisposable { private readonly int _bufferSize; private readonly List<(byte[] buffer, int position)> _buffers = new List<(byte[] buffer, int position)>(); private readonly List<(int bufferIndex, int offset, int length)> _sequences = new List<(int bufferIndex, int offset, int length)>(); private int _acquiredBufferIndex = -1; protected abstract byte AlgorithmIdentifier { get; } public abstract CompressionAlgorithm Algorithm { get; } protected CompressionEncoderBase(int bufferSize) { _bufferSize = bufferSize; } public Span GetSpan(int sizeHint) { var (buffer, position) = AcquireBuffer(sizeHint); return new Span(buffer, position, buffer.Length - position); } public Memory GetMemory(int sizeHint) { var (buffer, position) = AcquireBuffer(sizeHint); return new Memory(buffer, position, buffer.Length - position); } public Memory GetMemory() { return GetMemory(1); } private (byte[] buffer, int position) AcquireBuffer(int sizeHint) { if (_acquiredBufferIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Writing is already in progress."); for (var i = _buffers.Count - 1; i >= 0; i--) { (var buffer, int position) = _buffers[i]; if (buffer.Length - position >= sizeHint) { _acquiredBufferIndex = i; return (buffer, position); } } _acquiredBufferIndex = _buffers.Count; var nextBuffer = new byte[Math.Max(sizeHint, _bufferSize)]; _buffers.Add((nextBuffer, 0)); return (nextBuffer, 0); } public void Advance(int bytes) { if (bytes < 0) throw new ArgumentOutOfRangeException(nameof(bytes)); if (_acquiredBufferIndex < 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Writing is already completer."); var (buffer, position) = _buffers[_acquiredBufferIndex]; if (buffer.Length - position < bytes) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Attempt to write after the end of the memory buffer."); if (bytes > 0) { _buffers[_acquiredBufferIndex] = (buffer, position + bytes); (int bufferIndex, int offset, int length) lastSequence; if (_sequences.Count == 0 || (lastSequence = _sequences[^1]).bufferIndex != _acquiredBufferIndex) _sequences.Add((_acquiredBufferIndex, position, bytes)); else _sequences[^1] = (_acquiredBufferIndex, lastSequence.offset, lastSequence.length + bytes); } _acquiredBufferIndex = -1; } public void Reset() { if (_acquiredBufferIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Writing is in progress."); _sequences.Clear(); for (int i = 0; i < _buffers.Count; i++) _buffers[i] = (_buffers[i].buffer, 0); } public void Complete(ReadWriteBuffer pipeWriter) { if (_acquiredBufferIndex >= 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Writing is in progress."); /* * Compressed data consist of a sequence of compressed blocks. * * The structure of the block: * 1. CityHash checksum (16 bytes); * 2. Algorithm's identifier (1 byte); * 3. The size of the block without checksum (4 bytes); * 4. The size of data in the block without compression (4 bytes); * 5. The block of compressed data. */ var resultSequences = new List<(int bufferIndex, int offset, int length)>(_sequences.Count + 1); const int cityHashSize = 2 * sizeof(ulong); var header = new byte[cityHashSize + 1 + 2 * sizeof(int)]; var freeSequences = new Queue<(int bufferIndex, int offset, int length)>(); if (_buffers.Count > 0) { var lastBuffer = _buffers[^1]; if (lastBuffer.buffer.Length > lastBuffer.position) { freeSequences.Enqueue((_buffers.Count - 1, lastBuffer.position, lastBuffer.buffer.Length - lastBuffer.position)); _buffers[^1] = (lastBuffer.buffer, lastBuffer.buffer.Length); } } int readPosition = 0, sequenceIndex = 0; bool completed; do { if (resultSequences.Count > 0) { foreach (var sequence in resultSequences) freeSequences.Enqueue(sequence); resultSequences.Clear(); } completed = true; int writePosition = 0, rawSize = 0, encodedSize = 0; while (sequenceIndex < _sequences.Count) { var readSequence = _sequences[sequenceIndex]; if (readPosition == readSequence.length) { sequenceIndex++; readPosition = 0; freeSequences.Enqueue(readSequence); continue; } var count = ConsumeNext(_buffers[readSequence.bufferIndex].buffer, readSequence.offset + readPosition, readSequence.length - readPosition); readPosition += count; rawSize += count; if (readPosition < readSequence.length) { completed = false; break; } } if (rawSize == 0) break; (int bufferIndex, int offset, int length) currentSequence = (-1, 0, 0); while (true) { if (writePosition == currentSequence.length) { if (currentSequence.length > 0) resultSequences.Add(currentSequence); if (freeSequences.Count > 0) { currentSequence = freeSequences.Dequeue(); } else { currentSequence = (_buffers.Count, 0, _bufferSize); _buffers.Add((new byte[_bufferSize], _bufferSize)); } writePosition = 0; } int result = completed ? EncodeFinal(_buffers[currentSequence.bufferIndex].buffer, currentSequence.offset + writePosition, currentSequence.length - writePosition) : EncodeNext(_buffers[currentSequence.bufferIndex].buffer, currentSequence.offset + writePosition, currentSequence.length - writePosition); encodedSize += result; writePosition += result; if (writePosition < currentSequence.length) break; } if (writePosition > 0) resultSequences.Add((currentSequence.bufferIndex, currentSequence.offset, writePosition)); Span headerSpan = header; headerSpan[cityHashSize] = AlgorithmIdentifier; var success = BitConverter.TryWriteBytes(headerSpan.Slice(cityHashSize + 1), encodedSize + header.Length - cityHashSize); Debug.Assert(success); success = BitConverter.TryWriteBytes(headerSpan.Slice(cityHashSize + 1 + sizeof(int)), rawSize); Debug.Assert(success); var segments = new List>(resultSequences.Count + 1) {new ReadOnlyMemory(header)}; segments.AddRange(resultSequences.Select(s => new ReadOnlyMemory(_buffers[s.bufferIndex].buffer, s.offset, s.length))); var dataSegment = new SimpleReadOnlySequenceSegment(segments); var dataSequence = new ReadOnlySequence(dataSegment, 0, dataSegment.LastSegment, dataSegment.LastSegment.Memory.Length); var cityHash = CityHash.CityHash128(dataSequence.Slice(cityHashSize)); #if NET8_0_OR_GREATER var cityHashBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref cityHash, 1)); Debug.Assert(cityHashBytes.Length == 16); cityHashBytes.CopyTo(headerSpan); #else success = BitConverter.TryWriteBytes(headerSpan, cityHash.Low); Debug.Assert(success); success = BitConverter.TryWriteBytes(headerSpan.Slice(sizeof(ulong)), cityHash.High); Debug.Assert(success); #endif for (ReadOnlySequenceSegment? segment = dataSegment; segment != null; segment = segment.Next) { var sourceMem = segment.Memory; while (true) { var targetMem = pipeWriter.GetMemory(); if (sourceMem.Length > targetMem.Length) { sourceMem.Slice(0, targetMem.Length).CopyTo(targetMem); sourceMem = sourceMem.Slice(targetMem.Length); pipeWriter.ConfirmWrite(targetMem.Length); } else { sourceMem.CopyTo(targetMem); pipeWriter.ConfirmWrite(sourceMem.Length); break; } } } if (writePosition > 0) { // This entire sequence should be marked as free resultSequences[^1] = currentSequence; } } while (!completed); } protected abstract int ConsumeNext(byte[] source, int offset, int length); protected abstract int EncodeNext(byte[] target, int offset, int length); protected abstract int EncodeFinal(byte[] target, int offset, int length); public abstract void Dispose(); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseColumnReader.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { /// /// The interfaces for object capable of creating an internal column for . /// public interface IClickHouseColumnReader : IClickHouseColumnReaderBase { /// /// Creates a column for with the specified settings. /// /// The settings of the column. /// A column for . IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseColumnReaderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Buffers; namespace Octonica.ClickHouseClient.Protocol { /// /// The basic interface for objects capable of reading columnar data from a binary buffer. /// public interface IClickHouseColumnReaderBase { /// /// When implemented reads as much bytes as possible from the binary buffer. /// /// The binary buffer. /// The length of data that were read or if the provided buffer is too small. SequenceSize ReadNext(ReadOnlySequence sequence); /// /// When implemented reads the prefix specific to the column's type /// /// The binary buffer. /// /// The length of data that were read or if the provided buffer is too small. /// The prefix is counted for a single element, so the number of elements () /// can be either 0 or 1. /// SequenceSize ReadPrefix(ReadOnlySequence sequence) => new SequenceSize(0, 1); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseColumnWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Protocol { /// /// The base interface for an object capable of sequential writing column's data to a binary buffer. /// public interface IClickHouseColumnWriter { /// /// The name of the column to write data to. /// string ColumnName { get; } /// /// The full name of the ClickHouse type of the column. /// string ColumnType { get; } /// /// Writes next block of data to the target span. /// /// The buffer to write data to. /// The length of written data or if the provided buffer is too small. SequenceSize WriteNext(Span writeTo); /// /// Writes prefix specific to the column's type. /// /// The buffer to write data to. /// /// The length of written data or if the provided buffer is too small. /// The prefix is counted for a single element, so the number of elements () /// can be either 0 or 1. /// SequenceSize WritePrefix(Span writeTo) => new SequenceSize(0, 1); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseColumnWriterFactory.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal interface IClickHouseColumnWriterFactory { IClickHouseColumnWriter Create(int offset, int length); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseParameterValueWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Protocol { /// /// The base interface for a parameter's value writer. When implemented writes a parameter value in a binary format. /// /// /// Being a part of the ClickHouseClient's infrastructure, the interface is considered unstable. It can be changed between minor versions. /// public interface IClickHouseParameterValueWriter { /// /// The length of the parameter's value in bytes. /// If the length is less than zero the parameter will not be passed to the query. /// int Length { get; } /// /// When implemented writes the value to the . /// /// The buffer for writing the value. A caller of this method must ensure that the size of the buffer is not less than . /// The number of written bytes. int Write(Memory buffer); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseParameterWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Types; using System; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Octonica.ClickHouseClient.Protocol { /// /// The base interface for a value writer. When implemented writes a value as a ClickHouse parameter. /// /// The type of a value. /// /// Being a part of the ClickHouseClient's infrastructure, the interface is considered unstable. It can be changed between minor versions. /// public interface IClickHouseParameterWriter { /// /// Creates an instance of which encapsulates the value. /// /// The value of the parameter. /// The flag which indicates whether the value is a part of parameter's full value, e.g. an element of an array. /// >When this method returns, contains the instance of encapsulating the value. /// /// if the value can't be represented in a binary format. Otherwise returns and creates a . /// /// returned by this method instructs a parameter's writer to interpolate the value directly to the query text. bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter); /// /// Appends the value to the query as a ClickHouse literal. /// /// The builder of the SQL query /// The value for writing as a ClickHouse literal. /// The instance of the builder passed to the method (). StringBuilder Interpolate(StringBuilder queryBuilder, T value); /// /// Appends the value to the query. If required, the method appends type cast. /// For writing a value it invokes the callback . /// /// The builder of the SQL query /// The provider of type information. /// /// The function that appends an actual value to the query. It gets three arguments: the query string builder; the type of the /// parameter; the callback function for writing an external parts of the parameter. The type of the literal may differ from the type /// of the value. The function must return the same instance of the builder which it gets as an argument. /// /// The instance of the builder passed to the method (). StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClickHouseTableWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; namespace Octonica.ClickHouseClient.Protocol { internal interface IClickHouseTableWriter { string TableName { get; } int RowCount { get; } IReadOnlyList Columns { get; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IClientMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal interface IClientMessage { ClientMessageCode MessageCode { get; } void Write(ClickHouseBinaryProtocolWriter writer); } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/IServerMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal interface IServerMessage { ServerMessageCode MessageCode { get; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/Lz4CompressionDecoder.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using K4os.Compression.LZ4; namespace Octonica.ClickHouseClient.Protocol { internal sealed class Lz4CompressionDecoder : CompressionDecoderBase { protected override byte AlgorithmIdentifier => 0x82; public override CompressionAlgorithm Algorithm => CompressionAlgorithm.Lz4; public Lz4CompressionDecoder(int bufferSize) : base(bufferSize) { } protected override int Decode(ReadOnlySpan source, Span target) { return LZ4Codec.Decode(source, target); } public override void Dispose() { } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/Lz4CompressionEncoder.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using K4os.Compression.LZ4; using K4os.Compression.LZ4.Encoders; namespace Octonica.ClickHouseClient.Protocol { internal sealed class Lz4CompressionEncoder : CompressionEncoderBase { private readonly LZ4BlockEncoder _encoder; private readonly byte[] _compressedBuffer; private int _compressedSize; private int _compressedAvailable = -1; protected override byte AlgorithmIdentifier => 0x82; public override CompressionAlgorithm Algorithm => CompressionAlgorithm.Lz4; public Lz4CompressionEncoder(int bufferSize, int blockSize) : base(bufferSize) { _encoder = new LZ4BlockEncoder(LZ4Level.L10_OPT, blockSize); _compressedBuffer = new byte[LZ4Codec.MaximumOutputSize(blockSize)]; } protected override int ConsumeNext(byte[] source, int offset, int length) { _compressedAvailable = -1; return _encoder.Topup(source, offset, length); } protected override int EncodeNext(byte[] target, int offset, int length) { if (_compressedAvailable == 0) return 0; if (_compressedAvailable <0) { _compressedSize = _encoder.Encode(_compressedBuffer, 0, _compressedBuffer.Length, false); _compressedAvailable = _compressedSize; } var maxLen = Math.Min(length, _compressedAvailable); Array.Copy(_compressedBuffer, _compressedSize - _compressedAvailable, target, offset, maxLen); _compressedAvailable -= maxLen; return maxLen; } protected override int EncodeFinal(byte[] target, int offset, int length) { if (_compressedAvailable == 0) return 0; if (_compressedAvailable < 0) { _encoder.FlushAndEncode(_compressedBuffer, 0, _compressedBuffer.Length, false, out _compressedSize); _compressedAvailable = _compressedSize; } var maxLen = Math.Min(length, _compressedAvailable); Array.Copy(_compressedBuffer, _compressedSize - _compressedAvailable, target, offset, maxLen); _compressedAvailable -= maxLen; return maxLen; } public override void Dispose() { _encoder.Dispose(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/QueryKind.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal enum QueryKind { NoQuery = 0, InitialQuery = 1, SecondaryQuery = 2 } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/SequenceSize.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { /// /// Represents the size of a sequence of elements both in the number of bytes and in the number of elements. /// public readonly struct SequenceSize { /// /// The size of an empty sequence that has zero elements and zero bytes. /// public static readonly SequenceSize Empty = new SequenceSize(0, 0); /// /// The number of bytes in the sequence. /// public int Bytes { get; } /// /// The number of elements in the sequence. /// /// The sequence can contain zero elements with non-zero bytes when it has headers or other metadata. public int Elements { get; } /// /// Initializes a new instance of with the specified number of bytes and elements. /// /// The number of bytes in the sequence. /// The number of elements in the sequence. public SequenceSize(int bytes, int elements) { Bytes = bytes; Elements = elements; } /// /// Creates and returns a copy of with the specified number of bytes added to it. /// /// The number of bytes that should be added to the . /// The copy of with the specified number of bytes added to it. public SequenceSize AddBytes(int bytes) { return new SequenceSize(Bytes + bytes, Elements); } /// /// Creates and returns a copy of with the specified number of elements added to it. /// /// The number of elements that should be added to the . /// The copy of with the specified number of bytes added to it. public SequenceSize AddElements(int elements) { return new SequenceSize(Bytes, Elements + elements); } /// /// Creates and returns a copy of with the specified size added to it. /// The number of elements and the number of bytes are summed independently. /// /// The that should be added to this . /// The copy of with the specified size added to it. public SequenceSize Add(SequenceSize size) { return new SequenceSize(Bytes + size.Bytes, Elements + size.Elements); } /// /// Creates and returns a new instance representing the sum of two arguments. /// The number of elements and the number of bytes are summed independently. /// /// The first . /// The second . /// The new instance representing the sum of two arguments. public static SequenceSize operator +(SequenceSize x, SequenceSize y) { return new SequenceSize(x.Bytes + y.Bytes, x.Elements + y.Elements); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerDataMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerDataMessage : IServerMessage { public ServerMessageCode MessageCode { get; } public string? TempTableName { get; } private ServerDataMessage(ServerMessageCode messageCode, string? tempTableName) { MessageCode = messageCode; TempTableName = tempTableName; } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, ServerMessageCode messageCode, bool async, CancellationToken cancellationToken) { string? tempTableName = await reader.ReadString(async, cancellationToken); if (tempTableName == string.Empty) tempTableName = null; return new ServerDataMessage(messageCode, tempTableName); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerEndOfStreamMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerEndOfStreamMessage : IServerMessage { public static readonly ServerEndOfStreamMessage Instance = new ServerEndOfStreamMessage(); public ServerMessageCode MessageCode => ServerMessageCode.EndOfStream; private ServerEndOfStreamMessage() { } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerErrorMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerErrorMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.Error; public ClickHouseServerException Exception { get; } private ServerErrorMessage(ClickHouseServerException exception) { Exception = exception ?? throw new ArgumentNullException(nameof(exception)); } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, bool async, CancellationToken cancellationToken) { var errorCode = await reader.ReadInt32(async, cancellationToken); var name = await reader.ReadString(async, cancellationToken); var errorMessage = await reader.ReadString(async, cancellationToken); var stackTrace = await reader.ReadString(async, cancellationToken); bool hasNested = await reader.ReadBool(async, cancellationToken); ClickHouseServerException exception; if (hasNested) { var nested = await Read(reader, async, cancellationToken); exception = new ClickHouseServerException(errorCode, name, errorMessage, stackTrace, nested.Exception); } else { exception = new ClickHouseServerException(errorCode, name, errorMessage, stackTrace); } return new ServerErrorMessage(exception); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerHelloMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerHelloMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.Hello; public ClickHouseServerInfo ServerInfo { get; } private ServerHelloMessage(ClickHouseServerInfo serverInfo) { ServerInfo = serverInfo ?? throw new ArgumentNullException(nameof(serverInfo)); } public static async Task Read(ClickHouseBinaryProtocolReader reader, int protocolRevision, bool async, CancellationToken cancellationToken) { var serverName = await reader.ReadString(async, cancellationToken); var mj = await reader.Read7BitInt32(async, cancellationToken); var mr = await reader.Read7BitInt32(async, cancellationToken); var rv = await reader.Read7BitInt32(async, cancellationToken); if (rv < ClickHouseProtocolRevisions.MinSupportedRevision) { throw new ClickHouseException( ClickHouseErrorCodes.ProtocolRevisionNotSupported, $"The revision {rv} of ClickHouse server is not supported. Minimal supported revision is {ClickHouseProtocolRevisions.MinSupportedRevision}."); } var tz = await reader.ReadString(async, cancellationToken); var displayName = await reader.ReadString(async, cancellationToken); var versionPatch = await reader.Read7BitInt32(async, cancellationToken); var serverVersion = new ClickHouseVersion(mj, mr, versionPatch); var negotiatedRevision = Math.Min(rv, protocolRevision); List? complexityRules = null; if (negotiatedRevision >= ClickHouseProtocolRevisions.MinRevisionWithPasswordComplexityRules) { var rulesCount = await reader.Read7BitInt32(async, cancellationToken); if (rulesCount > 0) { complexityRules = new List(rulesCount); for (int i = 0; i < rulesCount; i++) { var pattern = await reader.ReadString(async, cancellationToken); var message = await reader.ReadString(async, cancellationToken); var rule = new ClickHousePasswordComplexityRule(pattern, message); complexityRules.Add(rule); } } } if (negotiatedRevision >= ClickHouseProtocolRevisions.MinRevisionWithInterserverSecretV2) await reader.SkipBytes(8, async, cancellationToken); // nonce var serverInfo = new ClickHouseServerInfo(serverName, serverVersion, serverRevision: rv, revision: negotiatedRevision, tz, displayName, complexityRules?.AsReadOnly()); return new ServerHelloMessage(serverInfo); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerMessageCode.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal enum ServerMessageCode { Hello = 0, Data = 1, Error = 2, Progress = 3, Pong = 4, EndOfStream = 5, ProfileInfo = 6, Totals = 7, Extremes = 8, TableStatusResponse = 9, Log = 10, TableColumns = 11, PartUuids = 12, ReadTaskRequest = 13, ProfileEvents = 14, MergeTreeAllRangesAnnouncement = 15, MergeTreeReadTaskRequest = 16, TimezoneUpdate = 17 } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerPongMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerPongMessage : IServerMessage { public static readonly ServerPongMessage Instance = new ServerPongMessage(); public ServerMessageCode MessageCode => ServerMessageCode.Pong; private ServerPongMessage() { } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerProfileInfoMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerProfileInfoMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.ProfileInfo; public ulong Rows { get; } public ulong Blocks { get; } public ulong Bytes { get; } public bool LimitApplied { get; } public ulong RowsBeforeLimit { get; } public bool CalculatedRowsBeforeLimit { get; } private ServerProfileInfoMessage(ulong rows, ulong blocks, ulong bytes, bool limitApplied, ulong rowsBeforeLimit, bool calculatedRowsBeforeLimit) { Rows = rows; Blocks = blocks; Bytes = bytes; LimitApplied = limitApplied; RowsBeforeLimit = rowsBeforeLimit; CalculatedRowsBeforeLimit = calculatedRowsBeforeLimit; } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, bool async, CancellationToken cancellationToken) { ulong rows = await reader.Read7BitUInt64(async, cancellationToken); ulong blocks = await reader.Read7BitUInt64(async, cancellationToken); ulong bytes = await reader.Read7BitUInt64(async, cancellationToken); bool limitApplied = await reader.ReadBool(async, cancellationToken); ulong rowsBeforeLimit = await reader.Read7BitUInt64(async, cancellationToken); bool calculatedRowsBeforeLimit = await reader.ReadBool(async, cancellationToken); return new ServerProfileInfoMessage(rows, blocks, bytes, limitApplied, rowsBeforeLimit, calculatedRowsBeforeLimit); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerProgressMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020, 2023, 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerProgressMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.Progress; public ClickHouseQueryExecutionProgress ExecutionProgress { get; } private ServerProgressMessage(ulong rows, ulong bytes, ulong totalRows, ulong totalBytes, ulong writtenRows, ulong writtenBytes, ulong elapsedNanoseconds) { ExecutionProgress = new ClickHouseQueryExecutionProgress(rows, bytes, totalRows, totalBytes, writtenRows, writtenBytes, elapsedNanoseconds); } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, int protocolRevision, bool async, CancellationToken cancellationToken) { ulong rows = await reader.Read7BitUInt64(async, cancellationToken); ulong bytes = await reader.Read7BitUInt64(async, cancellationToken); ulong totalRows = await reader.Read7BitUInt64(async, cancellationToken); ulong totalBytes = 0; if (protocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithTotalBytesInProgress) totalBytes = await reader.Read7BitUInt64(async, cancellationToken); ulong writtenRows = await reader.Read7BitUInt64(async, cancellationToken); ulong writtenBytes = await reader.Read7BitUInt64(async, cancellationToken); ulong elapsedNanoseconds = 0; if (protocolRevision >= ClickHouseProtocolRevisions.MinRevisionWithServerQueryTimeInProgress) elapsedNanoseconds = await reader.Read7BitUInt64(async, cancellationToken); return new ServerProgressMessage(rows, bytes, totalRows, totalBytes, writtenRows, writtenBytes, elapsedNanoseconds); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerTableColumnsMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerTableColumnsMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.TableColumns; public string Columns { get; } private ServerTableColumnsMessage(string columns) { Columns = columns; } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, bool async, CancellationToken cancellationToken) { await reader.ReadString(async, cancellationToken); var columns = await reader.ReadString(async, cancellationToken); return new ServerTableColumnsMessage(columns); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/ServerTimeZoneUpdateMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Threading.Tasks; using System.Threading; namespace Octonica.ClickHouseClient.Protocol { internal sealed class ServerTimeZoneUpdateMessage : IServerMessage { public ServerMessageCode MessageCode => ServerMessageCode.TimezoneUpdate; public string Timezone { get; } public ServerTimeZoneUpdateMessage(string timezone) { Timezone = timezone; } public static async ValueTask Read(ClickHouseBinaryProtocolReader reader, bool async, CancellationToken cancellationToken) { var timezone = await reader.ReadString(async, cancellationToken); return new ServerTimeZoneUpdateMessage(timezone); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/StateCodes.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal static class StateCodes { public const int Complete = 2; } } ================================================ FILE: src/Octonica.ClickHouseClient/Protocol/UnknownServerMessage.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Protocol { internal sealed class UnknownServerMessage : IServerMessage { public ServerMessageCode MessageCode { get; } public UnknownServerMessage(ServerMessageCode messageCode) { MessageCode = messageCode; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class ArrayTableColumn : IClickHouseTableColumn { private readonly IClickHouseTableColumn _column; private readonly List<(int offset, int length)> _ranges; public int RowCount => _ranges.Count; public ArrayTableColumn(IClickHouseTableColumn column, List<(int offset, int length)> ranges) { _column = column ?? throw new ArgumentNullException(nameof(column)); _ranges = ranges ?? throw new ArgumentNullException(nameof(ranges)); } public bool IsNull(int index) { return false; } public object GetValue(int index) { var range = _ranges[index]; var result = new object?[range.length]; if (result.Length == 0) return result; for (int i = 0; i < result.Length; i++) { if (_column.IsNull(range.offset + i)) result[i] = null; else result[i] = _column.GetValue(range.offset + i); } return result; } public IClickHouseTableColumn? TryReinterpret() { Type? elementType; var type = typeof(T); if (!type.IsArray || (elementType = type.GetElementType()) == null) return null; return (IClickHouseTableColumn?) TypeDispatcher.Dispatch(elementType, new ArrayTableColumnTypeDispatcher(_column, _ranges)); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } } internal sealed class ArrayTableColumn : IClickHouseTableColumn, IClickHouseArrayTableColumn { private readonly IClickHouseTableColumn _column; private readonly List<(int offset, int length)> _ranges; public int RowCount => _ranges.Count; public TElement[] DefaultValue { get; } public ArrayTableColumn(IClickHouseTableColumn column, List<(int offset, int length)> ranges) { _column = column ?? throw new ArgumentNullException(nameof(column)); _ranges = ranges ?? throw new ArgumentNullException(nameof(ranges)); DefaultValue = Array.Empty(); } public bool IsNull(int index) { return false; } public TElement[] GetValue(int index) { var range = _ranges[index]; if (range.length == 0) return Array.Empty(); var result = new TElement[range.length]; for (int i = 0; i < result.Length; i++) result[i] = _column.GetValue(range.offset + i); return result; } public IClickHouseTableColumn? TryReinterpret() { Type? elementType; var type = typeof(T); if (!type.IsArray || (elementType = type.GetElementType()) == null) return null; return (IClickHouseTableColumn?) TypeDispatcher.Dispatch(elementType, new ArrayTableColumnTypeDispatcher(_column, _ranges)); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { var reinterpretedColumn = _column as IClickHouseTableColumn ?? _column.TryReinterpret(); if (reinterpretedColumn == null) return null; return new ReinterpretedArrayTableColumn(this, new ArrayTableColumn(reinterpretedColumn, _ranges)); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public int CopyTo(int index, Span buffer, int dataOffset) { var range = _ranges[index]; if (dataOffset < 0 || dataOffset > range.length) throw new ArgumentOutOfRangeException(nameof(dataOffset)); var length = Math.Min(range.length - dataOffset, buffer.Length); for (int i = 0; i < length; i++) buffer[dataOffset + i] = _column.GetValue(range.offset + i); return length; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } internal sealed class ArrayTableColumnTypeDispatcher : ITypeDispatcher { private readonly IClickHouseTableColumn _column; private readonly List<(int offset, int length)> _ranges; public ArrayTableColumnTypeDispatcher(IClickHouseTableColumn column, List<(int offset, int length)> ranges) { _column = column ?? throw new ArgumentNullException(nameof(column)); _ranges = ranges; } public IClickHouseTableColumn? Dispatch() { var reinterpretedColumn = _column as IClickHouseTableColumn ?? _column.TryReinterpret(); if (reinterpretedColumn == null) return null; var reinterpretedArray = new ArrayTableColumn(reinterpretedColumn, _ranges); return reinterpretedArray; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ArrayTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class ArrayTypeInfo : IClickHouseColumnTypeInfo { private readonly IClickHouseColumnTypeInfo? _elementTypeInfo; public string ComplexTypeName { get; } public string TypeName { get; } = "Array"; public int GenericArgumentsCount => _elementTypeInfo == null ? 0 : 1; public ArrayTypeInfo() { ComplexTypeName = TypeName; } private ArrayTypeInfo(IClickHouseColumnTypeInfo elementTypeInfo) { _elementTypeInfo = elementTypeInfo ?? throw new ArgumentNullException(nameof(elementTypeInfo)); ComplexTypeName = $"{TypeName}({_elementTypeInfo.ComplexTypeName})"; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new ArrayColumnReader(rowCount, _elementTypeInfo); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"Custom serialization for {ComplexTypeName} is not supported by ClickHouseClient."); return CreateColumnReader(rowCount); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new ArraySkippingColumnReader(rowCount, _elementTypeInfo); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"Custom serialization for {ComplexTypeName} is not supported by ClickHouseClient."); return CreateSkippingColumnReader(rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var rowType = typeof(T); Type? elementType = null; if (rowType.IsArray) { var rank = rowType.GetArrayRank(); if (rank > 1) { elementType = rowType.GetElementType()!; Debug.Assert(elementType != null); var listAdapterInfo = MultiDimensionalArrayReadOnlyListAdapter.Dispatch(elementType, rowType.GetArrayRank()); var mdaDispatcher = new MultiDimensionalArrayColumnWriterDispatcher( columnName, (IReadOnlyList) rows, columnSettings, _elementTypeInfo, listAdapterInfo.createList); return TypeDispatcher.Dispatch(listAdapterInfo.listElementType, mdaDispatcher); } } foreach (var genericItf in rowType.GetInterfaces().Where(itf => itf.IsGenericType)) { if (genericItf.GetGenericTypeDefinition() != typeof(IReadOnlyList<>)) continue; if (elementType == null) { elementType = genericItf.GetGenericArguments()[0]; } else { var elementTypeCandidate = genericItf.GetGenericArguments()[0]; if (elementType.IsAssignableFrom(elementTypeCandidate)) elementType = elementTypeCandidate; else if (!elementTypeCandidate.IsAssignableFrom(elementType)) throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"Can't detect a type of the array's element. Candidates are: \"{elementType}\" and \"{elementTypeCandidate}\"."); } } ArrayColumnWriterDispatcherBase dispatcher; if (elementType == null) { var rowGenericTypeDef = rowType.GetGenericTypeDefinition(); if (rowGenericTypeDef == typeof(ReadOnlyMemory<>)) { elementType = rowType.GetGenericArguments()[0]!; dispatcher = new ReadOnlyColumnWriterDispatcher(columnName, rows, columnSettings, _elementTypeInfo); } else if (rowGenericTypeDef == typeof(Memory<>)) { elementType = rowType.GetGenericArguments()[0]!; dispatcher = new MemoryColumnWriterDispatcher(columnName, rows, columnSettings, _elementTypeInfo); } else { throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"Can't detect a type of the array's element. The type \"{typeof(T)}\" doesn't implement \"{typeof(IReadOnlyList<>)}\"."); } } else { dispatcher = new ArrayColumnWriterDispatcher(columnName, rows, columnSettings, _elementTypeInfo); } try { return TypeDispatcher.Dispatch(elementType, dispatcher); } catch (ClickHouseException ex) when (ex.ErrorCode == ClickHouseErrorCodes.TypeNotSupported) { throw new ClickHouseException(ex.ErrorCode, $"The type \"{rowType}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\". See the inner exception for details.", ex); } } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (options.Count > 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); var elementTypeInfo = typeInfoProvider.GetTypeInfo(options[0]); return new ArrayTypeInfo(elementTypeInfo); } public Type GetFieldType() { return _elementTypeInfo == null ? typeof(object[]) : _elementTypeInfo.GetFieldType().MakeArrayType(); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Array; } public IClickHouseTypeInfo GetGenericArgument(int index) { if (_elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); if (index != 0) throw new IndexOutOfRangeException(); return _elementTypeInfo; } public IClickHouseParameterWriter CreateParameterWriter() { if (_elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{ComplexTypeName}\" does not allow null values."); if (type == typeof(string)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); Type? elementType = null; foreach (var itf in type.GetInterfaces()) { if (!itf.IsGenericType) continue; var typeDef = itf.GetGenericTypeDefinition(); if (typeDef == typeof(ICollection<>) || typeDef == typeof(IReadOnlyCollection<>)) { var genericArg = itf.GetGenericArguments()[0]; if (elementType == null || elementType.IsAssignableFrom(genericArg)) { elementType = genericArg; } else if (!genericArg.IsAssignableFrom(elementType)) { throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"Can't detect a type of the array's element. Candidates are: \"{elementType}\" and \"{genericArg}\"."); } } } ArrayParameterWriterDispatcher dispatcher; if (elementType == null) { int arrayRank; if (!type.IsArray || (arrayRank = type.GetArrayRank()) == 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); var elementTypeInfo = _elementTypeInfo; for (int i = 1; i < arrayRank; i++) { if (elementTypeInfo is NullableTypeInfo nti) { elementTypeInfo = nti.UnderlyingType; --i; } else if (elementTypeInfo is ArrayTypeInfo ati) { elementTypeInfo = ati._elementTypeInfo; } else { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The multidimensional array value can not be converted type \"{ComplexTypeName}\": dimension number mismatch."); } if (elementTypeInfo == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); } elementType = type.GetElementType(); Debug.Assert( elementType != null ); dispatcher = new ArrayParameterWriterDispatcher(this, elementTypeInfo, true); } else { dispatcher = new ArrayParameterWriterDispatcher(this, _elementTypeInfo, false); } var writer = TypeDispatcher.Dispatch(elementType, dispatcher); return writer; } private abstract class ArrayColumnReaderBase : IClickHouseColumnReaderBase where TElementColumnReader : class, IClickHouseColumnReaderBase { private readonly int _rowCount; // A part of the inner column's header exposed outside of the beginning of the current column private byte[]? _prefix; private int _elementPosition; protected IClickHouseColumnTypeInfo ElementType { get; } protected List<(int offset, int length)> Ranges { get; } protected TElementColumnReader? ElementColumnReader { get; private set; } protected int Position { get; private set; } public ArrayColumnReaderBase(int rowCount, IClickHouseColumnTypeInfo elementType) { _rowCount = rowCount; ElementType = elementType; Ranges = new List<(int offset, int length)>(_rowCount); } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { // We can't read the prefix right now, so we keep it in the buffer until we get the real reader var skippingReader = ElementType.CreateSkippingColumnReader(0); var updSeq = sequence; if (_prefix != null) { var segment = new SimpleReadOnlySequenceSegment(_prefix, updSeq); updSeq = new ReadOnlySequence(segment, 0, segment.LastSegment, segment.LastSegment.Memory.Length); } var result = skippingReader.ReadPrefix(updSeq); var bufferSize = result.Bytes; if (_prefix != null) result = result.AddBytes(-_prefix.Length); if (result.Bytes < 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Byte offset calculation error. The length of the prefix is negative."); if (result.Bytes == 0) return result; Array.Resize(ref _prefix, bufferSize); updSeq.Slice(0, bufferSize).CopyTo(_prefix); return result; } SequenceSize IClickHouseColumnReaderBase.ReadNext(ReadOnlySequence sequence) { if (Position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); int bytesCount = 0; var slice = sequence; if (ElementColumnReader == null) { var totalLength = Ranges.Aggregate((ulong)0, (acc, r) => acc + (ulong)r.length); Span sizeSpan = stackalloc byte[sizeof(ulong)]; for (int i = Ranges.Count; i < _rowCount; i++) { if (slice.Length < sizeSpan.Length) return new SequenceSize(bytesCount, 0); ulong length; if (slice.FirstSpan.Length >= sizeSpan.Length) { length = BitConverter.ToUInt64(slice.FirstSpan); } else { slice.Slice(0, sizeSpan.Length).CopyTo(sizeSpan); length = BitConverter.ToUInt64(sizeSpan); } slice = slice.Slice(sizeSpan.Length); bytesCount += sizeSpan.Length; var offset = checked((int)totalLength); var rangeLength = checked((int)(length - totalLength)); Ranges.Add((offset, rangeLength)); totalLength = length; } ElementColumnReader = CreateElementColumnReader(checked((int)totalLength)); if (totalLength == 0) { // Special case for an empty array var result = new SequenceSize(bytesCount, _rowCount - Position); Position = _rowCount; return result; } } if (_prefix!=null) { var prefixSize = ElementColumnReader.ReadPrefix(new ReadOnlySequence(_prefix)); if (prefixSize.Bytes == 0 && prefixSize.Elements == 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Failed to read the column prefix."); if (prefixSize.Bytes != _prefix.Length) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. The column prefix' size is {_prefix.Length}, but the number of consumed bytes is {prefixSize.Bytes}."); if (prefixSize.Elements != 1) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Received an unexpected number of column prefixes: {prefixSize.Elements}."); _prefix = null; } var elementsSize = ElementColumnReader.ReadNext(slice); _elementPosition += elementsSize.Elements; var elementsCount = 0; while (Position < _rowCount) { var currentRange = Ranges[Position]; if (currentRange.length + currentRange.offset > _elementPosition) break; ++elementsCount; ++Position; } return new SequenceSize(bytesCount + elementsSize.Bytes, elementsCount); } protected abstract TElementColumnReader CreateElementColumnReader(int totalLength); } private sealed class ArrayColumnReader : ArrayColumnReaderBase, IClickHouseColumnReader { public ArrayColumnReader(int rowCount, IClickHouseColumnTypeInfo elementType) : base(rowCount, elementType) { } protected override IClickHouseColumnReader CreateElementColumnReader(int totalLength) { return ElementType.CreateColumnReader(totalLength); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { var elementColumnReader = ElementColumnReader ?? ElementType.CreateColumnReader(0); var column = elementColumnReader.EndRead(settings); var ranges = Position == Ranges.Count ? Ranges : Ranges.Take(Position).ToList(); if (!column.TryDipatch(new ArrayTableColumnDipatcher(ranges), out var result)) result = new ArrayTableColumn(column, ranges); return result; } } private sealed class ArraySkippingColumnReader : ArrayColumnReaderBase { public ArraySkippingColumnReader(int rowCount, IClickHouseColumnTypeInfo elementType) : base(rowCount, elementType) { } protected override IClickHouseColumnReaderBase CreateElementColumnReader(int totalLength) { return ElementType.CreateSkippingColumnReader(totalLength); } } private sealed class ArrayColumnWriterDispatcher : ArrayColumnWriterDispatcherBase { public ArrayColumnWriterDispatcher(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo elementTypeInfo) : base(columnName, rows, columnSettings, elementTypeInfo) { } protected override ArrayLinearizedList ToList(object rows) { return new ArrayLinearizedList(new ReadOnlyCollectionList((IReadOnlyList?>) rows)); } } private sealed class MemoryColumnWriterDispatcher : ArrayColumnWriterDispatcherBase { public MemoryColumnWriterDispatcher(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo elementTypeInfo) : base(columnName, rows, columnSettings, elementTypeInfo) { } protected override ArrayLinearizedList ToList(object rows) { return new ArrayLinearizedList(new MemoryCollectionList((IReadOnlyList>) rows)); } } private sealed class ReadOnlyColumnWriterDispatcher : ArrayColumnWriterDispatcherBase { public ReadOnlyColumnWriterDispatcher(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo elementTypeInfo) : base(columnName, rows, columnSettings, elementTypeInfo) { } protected override ArrayLinearizedList ToList(object rows) { return new ArrayLinearizedList(new ReadOnlyMemoryCollectionList((IReadOnlyList>) rows)); } } private abstract class ArrayColumnWriterDispatcherBase : ITypeDispatcher { private readonly string _columnName; private readonly object _rows; private readonly ClickHouseColumnSettings? _columnSettings; private readonly IClickHouseColumnTypeInfo _elementTypeInfo; protected ArrayColumnWriterDispatcherBase(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo elementTypeInfo) { _columnName = columnName; _rows = rows; _columnSettings = columnSettings; _elementTypeInfo = elementTypeInfo; } public IClickHouseColumnWriter Dispatch() { var linearizedList = ToList(_rows); var elementColumnWriter = _elementTypeInfo.CreateColumnWriter(_columnName, linearizedList, _columnSettings); var columnType = $"Array({elementColumnWriter.ColumnType})"; return new ArrayColumnWriter(columnType, linearizedList, elementColumnWriter); } protected abstract ArrayLinearizedList ToList(object rows); } private sealed class MultiDimensionalArrayColumnWriterDispatcher : ITypeDispatcher { private readonly string _columnName; private readonly IReadOnlyList _rows; private readonly ClickHouseColumnSettings? _columnSettings; private readonly IClickHouseColumnTypeInfo _elementTypeInfo; private readonly Func _dispatchArray; public MultiDimensionalArrayColumnWriterDispatcher( string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo elementTypeInfo, Func dispatchArray) { _columnName = columnName; _rows = rows; _columnSettings = columnSettings; _elementTypeInfo = elementTypeInfo; _dispatchArray = dispatchArray; } public IClickHouseColumnWriter Dispatch() { var mappedRows = MappedReadOnlyList>.Map(_rows, arr => (IReadOnlyList) _dispatchArray(arr)); var linearizedList = new ArrayLinearizedList(new ReadOnlyCollectionList(mappedRows)); var elementColumnWriter = _elementTypeInfo.CreateColumnWriter(_columnName, linearizedList, _columnSettings); var columnType = $"Array({elementColumnWriter.ColumnType})"; return new ArrayColumnWriter(columnType, linearizedList, elementColumnWriter); } } private sealed class ArrayColumnWriter : IClickHouseColumnWriter { private readonly ArrayLinearizedList _rows; private readonly IClickHouseColumnWriter _elementColumnWriter; public string ColumnName { get; } public string ColumnType { get; } private int _headerPosition; private int _position; private int _elementPosition; public ArrayColumnWriter(string columnType, ArrayLinearizedList rows, IClickHouseColumnWriter elementColumnWriter) { _rows = rows; _elementColumnWriter = elementColumnWriter; ColumnName = elementColumnWriter.ColumnName; ColumnType = columnType; } SequenceSize IClickHouseColumnWriter.WritePrefix(Span writeTo) { return _elementColumnWriter.WritePrefix(writeTo); } public SequenceSize WriteNext(Span writeTo) { int bytesCount = 0; var span = writeTo; for (; _headerPosition < _rows.ListLengths.Count; _headerPosition++) { if (!BitConverter.TryWriteBytes(span, (ulong) _rows.ListLengths[_headerPosition])) return new SequenceSize(bytesCount, 0); span = span.Slice(sizeof(ulong)); bytesCount += sizeof(ulong); } if (_rows.Count == 0) { // There are no actual values, only an empty array or a bunch of empty arrays Debug.Assert(_position == 0); _position = _rows.ListLengths.Count; return new SequenceSize(bytesCount, _position); } var elementSize = _elementColumnWriter.WriteNext(span); _elementPosition += elementSize.Elements; var elementsCount = 0; while (_position < _rows.ListLengths.Count) { if (_rows.ListLengths[_position] > _elementPosition) break; ++elementsCount; ++_position; } return new SequenceSize(elementSize.Bytes + bytesCount, elementsCount); } } private sealed class ArrayLinearizedList : IReadOnlyList { private readonly ICollectionList _listOfLists; public List ListLengths { get; } public int Count { get; } public ArrayLinearizedList(ICollectionList listOfLists) { _listOfLists = listOfLists; ListLengths = new List(_listOfLists.Count); int offset = 0; foreach (var listLength in _listOfLists.GetListLengths()) { offset += listLength; ListLengths.Add(offset); } Count = offset; } public IEnumerator GetEnumerator() { return _listOfLists.GetItems().GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public T this[int index] { get { if (index < 0 || index >= Count) throw new IndexOutOfRangeException(); var listIndex = ListLengths.BinarySearch(index); int elementIndex; if (listIndex < 0) { listIndex = ~listIndex; if (listIndex == 0) { elementIndex = index; } else { elementIndex = index - ListLengths[listIndex - 1]; } } else { elementIndex = 0; listIndex++; } for (; listIndex < _listOfLists.Count; listIndex++) { if (elementIndex < 0) break; var listLength = _listOfLists.GetLength(listIndex); if (elementIndex < listLength) return _listOfLists[listIndex, elementIndex]; elementIndex = index - ListLengths[listIndex]; } throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error: data structure is corrupted."); } } } private sealed class ArrayTableColumnDipatcher : IClickHouseTableColumnDispatcher { private readonly List<(int offset, int length)> _ranges; public ArrayTableColumnDipatcher(List<(int offset, int length)> ranges) { _ranges = ranges; } public IClickHouseTableColumn Dispatch(IClickHouseTableColumn column) { return new ArrayTableColumn(column, _ranges); } } private sealed class ArrayParameterWriterDispatcher : ITypeDispatcher> { private readonly ArrayTypeInfo _arrayType; private readonly IClickHouseColumnTypeInfo _elementType; private readonly bool _isMultidimensional; public ArrayParameterWriterDispatcher(ArrayTypeInfo arrayType, IClickHouseColumnTypeInfo elementType, bool isMultidimensional) { _arrayType = arrayType; _elementType = elementType; _isMultidimensional = isMultidimensional; } public IClickHouseParameterWriter Dispatch() { var elementWriter = _elementType.CreateParameterWriter(); if (_isMultidimensional) return new MultidimensionalArralParameterWriter(_arrayType, elementWriter); return new ArrayParameterWriter(_arrayType, elementWriter); } } private sealed class MultidimensionalArralParameterWriter : IClickHouseParameterWriter { private readonly ArrayTypeInfo _arrayType; private readonly IClickHouseParameterWriter _elementWriter; public MultidimensionalArralParameterWriter(ArrayTypeInfo arrayType, IClickHouseParameterWriter elementWriter) { _arrayType = arrayType; _elementWriter = elementWriter; } public bool TryCreateParameterValueWriter(TArray value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { if (value is null) throw new ArgumentNullException(nameof(value)); var enumerator = ((IEnumerable)value).GetEnumerator(); if (!TryCreateElementWriters((Array)(object)value, enumerator, 0, out var elementWriters)) { valueWriter = null; return false; } valueWriter = new ArrayLiteralValueWriter(elementWriters); return true; } private bool TryCreateElementWriters(Array array, IEnumerator enumerator, int dimension, [NotNullWhen(true)] out List? writers) { var rankLength = array.GetLength(dimension); writers = new List(rankLength); if (dimension == array.Rank - 1) { for (var i = 0; i < rankLength; ++i) { if (!enumerator.MoveNext()) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error: unexpected iterator out of bound."); if (!_elementWriter.TryCreateParameterValueWriter((TElement)enumerator.Current!, true, out var elementWriter)) return false; writers.Add(elementWriter); } } else { var nextDimension = dimension + 1; for (var i = 0; i < rankLength; ++i) { if (!TryCreateElementWriters(array, enumerator, nextDimension, out var elementWriters)) return false; writers.Add(new ArrayLiteralValueWriter(elementWriters)); } } return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, TArray value) { if (value is null) throw new ArgumentNullException(nameof(value)); var enumerator = ((IEnumerable)value).GetEnumerator(); return Interpolate(queryBuilder, (Array)(object)value, enumerator, 0); } private StringBuilder Interpolate(StringBuilder queryBuilder, Array array, IEnumerator enumerator, int dimension) { var rankLength = array.GetLength(dimension); queryBuilder.Append('['); if (dimension == array.Rank - 1) { for (var i = 0; i < rankLength; ++i) { if (!enumerator.MoveNext()) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error: unexpected iterator out of bound."); if (i > 0) queryBuilder.Append(','); _elementWriter.Interpolate(queryBuilder, (TElement)enumerator.Current!); } } else { var nextDimension = dimension + 1; for (var i = 0; i < rankLength; ++i) { if (i > 0) queryBuilder.Append(','); Interpolate(queryBuilder, array, enumerator, nextDimension); } } return queryBuilder.Append(']'); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _elementWriter.Interpolate(queryBuilder, typeInfoProvider, (qb, typeInfo, writeElement) => { int rank = 1; var elementTypeInfo = _arrayType._elementTypeInfo; while (elementTypeInfo is ArrayTypeInfo elementArray) { elementTypeInfo = elementArray._elementTypeInfo; ++rank; } Debug.Assert(elementTypeInfo != null); if (elementTypeInfo.ComplexTypeName == typeInfo.ComplexTypeName) return writeValue(qb, _arrayType, FunctionHelper.Apply); var updArrayTypeInfo = new ArrayTypeInfo(typeInfo); for (int i = rank - 1; i > 0; i--) updArrayTypeInfo = new ArrayTypeInfo(updArrayTypeInfo); return writeValue(qb, updArrayTypeInfo, (qb2, realWrite) => { for (int i = 1; i <= rank; i++) qb2.AppendFormat(CultureInfo.InvariantCulture, "arrayMap(_elt{0} -> ", i); writeElement(qb2, b => b.AppendFormat(CultureInfo.InvariantCulture, "_elt{0}", rank)); for (int i = rank - 1; i > 0; i--) qb2.AppendFormat(CultureInfo.InvariantCulture, ", _elt{0})", i); qb2.Append(", "); realWrite(qb2); qb2.Append(')'); return qb2; }); }); } } private sealed class ArrayParameterWriter : IClickHouseParameterWriter { private readonly ArrayTypeInfo _type; private readonly IClickHouseParameterWriter _elementWriter; public ArrayParameterWriter(ArrayTypeInfo type, IClickHouseParameterWriter elementWriter) { _type = type; _elementWriter = elementWriter; } public bool TryCreateParameterValueWriter(TArray value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { if (value is null) throw new ArgumentNullException(nameof(value)); var elementWriters = new List(); foreach(var element in (IEnumerable)value) { if (!_elementWriter.TryCreateParameterValueWriter(element, true, out var elementWriter)) { valueWriter = null; return false; } elementWriters.Add(elementWriter); } valueWriter = new ArrayLiteralValueWriter(elementWriters); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, TArray value) { if (value is null) throw new ArgumentNullException(nameof(value)); var elementTypeInfo = _type._elementTypeInfo; Debug.Assert(elementTypeInfo != null); queryBuilder.Append('['); bool isFirst = true; foreach (var element in (IEnumerable)value) { if (isFirst) isFirst = false; else queryBuilder.Append(','); _elementWriter.Interpolate(queryBuilder, element); } return queryBuilder.Append(']'); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _elementWriter.Interpolate(queryBuilder, typeInfoProvider, (qb, typeInfo, writeElement) => { var elementTypeInfo = _type._elementTypeInfo; Debug.Assert(elementTypeInfo != null); if (elementTypeInfo.ComplexTypeName == typeInfo.ComplexTypeName) return writeValue(qb, _type, FunctionHelper.Apply); var updArrayTypeInfo = new ArrayTypeInfo(typeInfo); return writeValue(qb, updArrayTypeInfo, (qb2, realWrite) => { qb2.Append("arrayMap(_elt -> "); writeElement(qb2, b => b.Append("_elt")); qb2.Append(", "); realWrite(qb2); return qb2.Append(')'); }); }); } } private sealed class ArrayLiteralValueWriter : IClickHouseParameterValueWriter { private readonly List _elementWriters; public int Length { get; } public ArrayLiteralValueWriter(List elementWriters) { Length = 2 + // [] elementWriters.Aggregate(0, (l, w) => w.Length + l) + // length of elements Math.Max(0, elementWriters.Count - 1); // comas _elementWriters = elementWriters; } public int Write(Memory buffer) { int count = 0; buffer.Span[count++] = (byte)'['; bool isFirst = true; foreach (var writer in _elementWriters) { if (isFirst) isFirst = false; else buffer.Span[count++] = (byte)','; count += writer.Write(buffer.Slice(count)); } buffer.Span[count++] = (byte)']'; Debug.Assert(count == Length); return count; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/BigIntegerTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Numerics; namespace Octonica.ClickHouseClient.Types { internal sealed class BigIntegerTableColumn : IClickHouseTableColumn { private readonly byte[] _rawData; private readonly int _elementByteSize; private readonly bool _isUnsigned; public int RowCount { get; } public BigInteger DefaultValue => BigInteger.Zero; public BigIntegerTableColumn(byte[] rawData, int rowCount, int elementByteSize, bool isUnsigned) { _rawData = rawData; RowCount = rowCount; _elementByteSize = elementByteSize; _isUnsigned = isUnsigned; } public BigInteger GetValue(int index) { if (index < 0 || index >= RowCount) throw new IndexOutOfRangeException(); var slice = ((ReadOnlySpan)_rawData).Slice(index * _elementByteSize, _elementByteSize); return new BigInteger(slice, _isUnsigned); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public bool IsNull(int index) { return false; } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(BigInteger?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, this); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/BigIntegerTypeInfoBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Buffers; using System.Collections.Generic; using System.Globalization; using System.Diagnostics; using System.Numerics; namespace Octonica.ClickHouseClient.Types { internal abstract class BigIntegerTypeInfoBase : SimpleTypeInfo { int _elementByteSize; bool _isUnsigned; protected BigIntegerTypeInfoBase(string typeName, int elementByteSize, bool isUnsigned) : base(typeName) { _elementByteSize = elementByteSize; _isUnsigned = isUnsigned; } public sealed override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new BigIntegerColumnReader(rowCount, _elementByteSize, _isUnsigned); } public sealed override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(_elementByteSize, rowCount); } public sealed override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList? bigIntegerRows = null; if (type == typeof(BigInteger)) { bigIntegerRows = (IReadOnlyList)rows; } else if (type == typeof(ulong)) { bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); } else if (type == typeof(uint)) { bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); } else if (type == typeof(ushort)) { bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); } else if (type == typeof(byte)) { bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); } else if (!_isUnsigned) { if (type == typeof(long)) bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(int)) bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(short)) bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(sbyte)) bigIntegerRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); } if (bigIntegerRows == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new BigIntegerColumnWriter(columnName, ComplexTypeName, _elementByteSize, bigIntegerRows, _isUnsigned); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object? writer = null; if (type == typeof(BigInteger)) { if (_isUnsigned) { writer = new StringParameterWriter( this, bigIntegerValue => { if (bigIntegerValue.Sign < 0) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow negative BigInteger values."); return bigIntegerValue.ToString(CultureInfo.InvariantCulture).AsMemory(); }); } else { writer = StringParameterWriter.Create(this); } } else if (type == typeof(ulong)) { writer = StringParameterWriter.Create(this); } else if (type == typeof(uint)) { writer = StringParameterWriter.Create(this); } else if (type == typeof(ushort)) { writer = StringParameterWriter.Create(this); } else if (type == typeof(byte)) { writer = StringParameterWriter.Create(this); } else if (!_isUnsigned) { if (type == typeof(long)) writer = StringParameterWriter.Create(this); if (type == typeof(int)) writer = StringParameterWriter.Create(this); if (type == typeof(short)) writer = StringParameterWriter.Create(this); else if (type == typeof(sbyte)) writer = StringParameterWriter.Create(this); } if (writer == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public sealed override Type GetFieldType() { return typeof(BigInteger); } private sealed class BigIntegerColumnReader : IClickHouseColumnReader { private readonly byte[] _buffer; private readonly int _elementByteSize; private readonly bool _isUnsigned; private int _position; public BigIntegerColumnReader(int rowCount, int elementByteSize, bool isUnsigned) { if (rowCount == 0) _buffer = Array.Empty(); else _buffer = new byte[rowCount * elementByteSize]; _elementByteSize = elementByteSize; _isUnsigned = isUnsigned; } public SequenceSize ReadNext(ReadOnlySequence sequence) { var rowCount = _buffer.Length / _elementByteSize; if (_position >= rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var elementCount = (int)Math.Min(rowCount - _position, sequence.Length / _elementByteSize); var byteCount = elementCount * _elementByteSize; sequence.Slice(0, byteCount).CopyTo(((Span)_buffer).Slice(_position * _elementByteSize, byteCount)); _position += elementCount; return new SequenceSize(byteCount, elementCount); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return new BigIntegerTableColumn(_buffer, _position, _elementByteSize, _isUnsigned); } } private sealed class BigIntegerColumnWriter : StructureWriterBase { private readonly bool _isUnsigned; public BigIntegerColumnWriter(string columnName, string columnType, int elementSize, IReadOnlyList rows, bool isUnsigned) : base(columnName, columnType, elementSize, rows) { _isUnsigned = isUnsigned; } protected override void WriteElement(Span writeTo, in BigInteger value) { if (_isUnsigned && value.Sign < 0) throw new OverflowException($"A negative value can't be written to the column \"{ColumnName}\" of type \"{ColumnType}\"."); var byteCount = value.GetByteCount(_isUnsigned); if (byteCount > ElementSize) throw new OverflowException($"A value can't be written to the column \"{ColumnName}\" of type \"{ColumnType}\" because it's outside of supported range of values."); var success = value.TryWriteBytes(writeTo.Slice(0, byteCount), out var bytesWritten, _isUnsigned); Debug.Assert(success); Debug.Assert(byteCount == bytesWritten); if (byteCount < ElementSize) writeTo.Slice(byteCount, ElementSize - byteCount).Fill(_isUnsigned || value.Sign >= 0 ? (byte)0 : (byte)0xff); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/BoolTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2022, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class BoolTableColumn : IClickHouseTableColumn { private readonly ReadOnlyMemory _buffer; public int RowCount => _buffer.Length; public bool DefaultValue => false; public BoolTableColumn(ReadOnlyMemory buffer) { _buffer = buffer; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public bool GetValue(int index) { return _buffer.Span[index] != 0; } public bool IsNull(int index) { return false; } public IClickHouseTableColumn? TryReinterpret() { // Inherit type cast logic from UInt8 var uint8Column = new UInt8TableColumn(_buffer); var reinterpreted = uint8Column as IClickHouseTableColumn ?? uint8Column.TryReinterpret(); if (reinterpreted == null) return null; return new ReinterpretedTableColumn(this, reinterpreted); } public bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/BoolTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2022-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Types { internal sealed class BoolTypeInfo : SimpleTypeInfo { public BoolTypeInfo() : base("Bool") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new BoolReader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(byte), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { IReadOnlyList typedList; if (typeof(T) == typeof(bool)) { typedList = (IReadOnlyList)rows; } else if (typeof(T) == typeof(byte)) { // Some kind of a compatibility mode. Write bytes as bools. Any non-zero value is treated as true. typedList = MappedReadOnlyList.Map((IReadOnlyList)rows, b => b != 0); } else { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } return new BoolWriter(columnName, ComplexTypeName, typedList); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(bool)) writer = new SimpleParameterWriter(this, appendTypeCast: true, boolValue => boolValue ? (byte)1 : (byte)0); else if (type == typeof(byte)) writer = new SimpleParameterWriter(this, appendTypeCast: true); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Boolean; } public override Type GetFieldType() { return typeof(bool); } private sealed class BoolReader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public BoolReader(int rowCount) : base(sizeof(byte), rowCount) { } protected override byte ReadElement(ReadOnlySpan source) { return source[0]; } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new BoolTableColumn(buffer); } } private sealed class BoolWriter : StructureWriterBase { public BoolWriter(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(byte), rows) { } protected override byte Convert(bool value) { return value ? (byte)1 : (byte)0; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ClickHouseColumnReinterpreter.cs ================================================ #region License Apache 2.0 /* Copyright 2024, 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class ClickHouseColumnReinterpreter : ITypeDispatcher { private static readonly ClickHouseColumnReinterpreter Instance = new ClickHouseColumnReinterpreter(); private ClickHouseColumnReinterpreter() { } IClickHouseColumnReinterpreter ITypeDispatcher.Dispatch() { return ClickHouseColumnReinterpreter.Instance; } public static IClickHouseColumnReinterpreter Create(Type type) { return TypeDispatcher.Create(type).Dispatch(Instance); } public static ClickhouseColumnReinterpreterDispatcher CreateDispatcher(IConverterDispatcher dispatcher) { return new ClickhouseColumnReinterpreterDispatcher(dispatcher); } public static IClickHouseColumnReinterpreter Create(Func convert) { var srcType = typeof(T); var resType = typeof(TRes); Type implTypeDef; if (srcType.IsValueType) { srcType = Nullable.GetUnderlyingType(srcType) ?? srcType; if (resType.IsValueType) { resType = Nullable.GetUnderlyingType(resType) ?? resType; implTypeDef = typeof(ClickHouseColumnStructToStructReinterpreter<,>); } else { implTypeDef = typeof(ClickHouseColumnStructToObjReinterpreter<,>); } } else { if (resType.IsValueType) { resType = Nullable.GetUnderlyingType(resType) ?? resType; implTypeDef = typeof(ClickHouseColumnObjToStructReinterpreter<,>); } else { implTypeDef = typeof(ClickHouseColumnObjToObjReinterpreter<,>); } } var implType = implTypeDef.MakeGenericType(srcType, resType); var impl = Activator.CreateInstance(implType, convert); Debug.Assert(impl != null); return (IClickHouseColumnReinterpreter)impl; } public static TValue GuardNull(TValue? value) where TValue : struct { return value ?? throw new ClickHouseException( ClickHouseErrorCodes.CallbackError, "A callback function provided by the caller returned a null reference." + Environment.NewLine + "When configuring a data reader with a custom column reader, make sure that the callback function will not return null when its argument is not null."); } [return: NotNull] public static TValue GuardNull(TValue? value) where TValue : class { return value ?? throw new ClickHouseException( ClickHouseErrorCodes.CallbackError, "A callback function provided by the caller returned a null reference." + Environment.NewLine + "When configuring a data reader with a custom column reader, make sure that the callback function will not return null when its argument is not null."); } public static IClickHouseTableColumn Reinterpret(IClickHouseTableColumn? root, IClickHouseTableColumn column, Func convert) { if (column is IClickHouseReinterpretedTableColumn rc) return rc.Chain(convert); return new ReinterpretedTableColumn(root, column, convert); } } internal sealed class ClickhouseColumnReinterpreterDispatcher : IClickHouseTableColumnDispatcher, IConverterDispatcher { private readonly IConverterDispatcher _converterDispatcher; public ClickhouseColumnReinterpreterDispatcher(IConverterDispatcher converterDispatcher) { _converterDispatcher = converterDispatcher; } public IClickHouseColumnReinterpreter? Dispatch(IClickHouseTableColumn column) { return Dispatch(); } public IClickHouseColumnReinterpreter? Dispatch() { return _converterDispatcher.Dispatch(this); } public IClickHouseColumnReinterpreter Dispatch(Func convert) { if (typeof(TFrom) == typeof(object)) return new ClickHouseObjectColumnReinterpreter((Func)(object)convert); return ClickHouseColumnReinterpreter.Create(convert); } public IClickHouseColumnReinterpreter? DispatchNoConvert() { return null; } } internal sealed class ClickHouseColumnReinterpreter : IClickHouseColumnReinterpreter { public static ClickHouseColumnReinterpreter Instance = new ClickHouseColumnReinterpreter(); public Type BuiltInConvertToType => typeof(T); public Type? ExternalConvertToType => null; private ClickHouseColumnReinterpreter() { } public IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { return (column as IClickHouseTableColumn) ?? column.TryReinterpret(); } } internal sealed class ClickHouseObjectColumnReinterpreter : IClickHouseColumnReinterpreter { private readonly Func _convert; public Type? BuiltInConvertToType => null; public Type ExternalConvertToType => typeof(T); public ClickHouseObjectColumnReinterpreter(Func convert) { _convert = Guard(convert); } public IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { if (column is IClickHouseReinterpretedTableColumn rc) return rc.Chain(_convert); return new ReinterpretedObjectTableColumn(column, _convert); } private static Func Guard(Func convert) { return v => convert(v ?? DBNull.Value) ?? (T)(object)DBNull.Value; } } internal abstract class ClickHouseColumnReinterpreter : IClickHouseColumnReinterpreter { public Type BuiltInConvertToType => typeof(T); public Type ExternalConvertToType => typeof(TRes); public abstract IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column); } internal sealed class ClickHouseColumnObjToObjReinterpreter : ClickHouseColumnReinterpreter where T : class where TRes : class { private readonly Func _convert; public ClickHouseColumnObjToObjReinterpreter(Func convert) { _convert = convert; } public override IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { if (column is NullableObjTableColumn nc) return nc.ReinterpretAsObj(Guard(_convert)); if (column is IClickHouseTableColumn c) return ClickHouseColumnReinterpreter.Reinterpret(null, c, GuardNullable(_convert)); var reinterpretedColumn = column.TryReinterpret(); if (reinterpretedColumn == null) return null; if (reinterpretedColumn is NullableObjTableColumn nrc) nrc.ReinterpretAsObj(Guard(_convert)); return ClickHouseColumnReinterpreter.Reinterpret(column, reinterpretedColumn, GuardNullable(_convert)); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func GuardNullable(Func convert) { return v => v == null ? null! : ClickHouseColumnReinterpreter.GuardNull(convert(v)); } } internal sealed class ClickHouseColumnObjToStructReinterpreter : ClickHouseColumnReinterpreter where T : class where TRes : struct { private readonly Func? _convert0; private readonly Func? _convert1; public ClickHouseColumnObjToStructReinterpreter(Func convert) { _convert1 = convert; } public ClickHouseColumnObjToStructReinterpreter(Func convert) { _convert0 = convert; } public override IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { if (column is NullableObjTableColumn nullableColumn) return nullableColumn.ReinterpretAsStruct(GetConvert()); IClickHouseTableColumn? rootColumn = null; var reinterpretedColumn = column as IClickHouseTableColumn; if (reinterpretedColumn == null) rootColumn = column; reinterpretedColumn = column.TryReinterpret(); if (reinterpretedColumn == null) return null; if (reinterpretedColumn is NullableObjTableColumn nullableReinterpretedColumn) return nullableReinterpretedColumn.ReinterpretAsStruct(GetConvert()); if (_convert0 != null) { // The caller explixitly requested the conversion to a nullable struct return ClickHouseColumnReinterpreter.Reinterpret(rootColumn, reinterpretedColumn, GuardNullable(_convert0)); } // It's safe to assume that the column is non-nullable return ClickHouseColumnReinterpreter.Reinterpret(rootColumn, reinterpretedColumn, GetConvert()); } private Func GetConvert() { if (_convert1 != null) return _convert1; if (_convert0 != null) return Guard(_convert0); throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Ivalid object state. If you see this message, please, report a bug."); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func GuardNullable(Func convert) { return v => v == null ? (TRes?)null : ClickHouseColumnReinterpreter.GuardNull(convert(v)); } } internal sealed class ClickHouseColumnStructToObjReinterpreter : ClickHouseColumnReinterpreter where T : struct where TRes : class { private readonly Func? _convert0; private readonly Func? _convert1; public ClickHouseColumnStructToObjReinterpreter(Func convert) { _convert1 = convert; } public ClickHouseColumnStructToObjReinterpreter(Func convert) { _convert0 = convert; } public override IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { if (column is IClickHouseTableColumn structColumn) return ClickHouseColumnReinterpreter.Reinterpret(null, structColumn, GetConvert()); if (column is IClickHouseTableColumn nullableStructColumn) return ClickHouseColumnReinterpreter.Reinterpret(null, nullableStructColumn, GetConvertNullable()); IClickHouseTableColumn? reinterpretedNullableColumn; var reinterpretedStructColumn = column.TryReinterpret(); if (reinterpretedStructColumn is NullableStructTableColumnNotNullableAdapter adapter) { reinterpretedNullableColumn = adapter.Unguard(); } else if (reinterpretedStructColumn != null) { return ClickHouseColumnReinterpreter.Reinterpret(column, reinterpretedStructColumn, GetConvert()); } else { reinterpretedNullableColumn = column.TryReinterpret(); } if (reinterpretedNullableColumn != null) { return ClickHouseColumnReinterpreter.Reinterpret(column, reinterpretedNullableColumn, GetConvertNullable()); } return null; } private Func GetConvert() { if (_convert1 != null) return Guard(_convert1); if (_convert0 != null) return Guard(_convert0); throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Ivalid object state. If you see this message, please, report a bug."); } private Func GetConvertNullable() { if (_convert1 != null) return GuardNullable(_convert1); if (_convert0 != null) return GuardNullable(_convert0); throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Ivalid object state. If you see this message, please, report a bug."); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func GuardNullable(Func convert) { return v => v == null ? null! : ClickHouseColumnReinterpreter.GuardNull(convert(v.Value)); } private static Func GuardNullable(Func convert) { return v => v == null ? null! : ClickHouseColumnReinterpreter.GuardNull(convert(v)); } } internal sealed class ClickHouseColumnStructToStructReinterpreter : ClickHouseColumnReinterpreter where T : struct where TRes : struct { private readonly Func? _convert00; private readonly Func? _convert01; private readonly Func? _convert10; private readonly Func? _convert11; public ClickHouseColumnStructToStructReinterpreter(Func convert) { _convert00 = convert; } public ClickHouseColumnStructToStructReinterpreter(Func convert) { _convert01 = convert; } public ClickHouseColumnStructToStructReinterpreter(Func convert) { _convert10 = convert; } public ClickHouseColumnStructToStructReinterpreter(Func convert) { _convert11 = convert; } public override IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { if (column is IClickHouseTableColumn structColumn) return ClickHouseColumnReinterpreter.Reinterpret(null, structColumn, GetConvert()); if (column is IClickHouseTableColumn nullableStructColumn) return ClickHouseColumnReinterpreter.Reinterpret(null, nullableStructColumn, GetConvertNullable()); IClickHouseTableColumn? reinterpretedNullableColumn; var reinterpretedStructColumn = column.TryReinterpret(); if (reinterpretedStructColumn is NullableStructTableColumnNotNullableAdapter adapter) { reinterpretedNullableColumn = adapter.Unguard(); } else if (reinterpretedStructColumn != null) { return ClickHouseColumnReinterpreter.Reinterpret(column, reinterpretedStructColumn, GetConvert()); } else { reinterpretedNullableColumn = column.TryReinterpret(); } if (reinterpretedNullableColumn != null) { return ClickHouseColumnReinterpreter.Reinterpret(column, reinterpretedNullableColumn, GetConvertNullable()); } return null; } private Func GetConvert() { if (_convert11 != null) return _convert11; if (_convert00 != null) return Guard(_convert00); if (_convert01 != null) return Guard(_convert01); if (_convert10 != null) return Guard(_convert10); throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Ivalid object state. If you see this message, please, report a bug."); } private Func GetConvertNullable() { if (_convert11 != null) return GuardNullable(_convert11); if (_convert00 != null) return GuardNullable(_convert00); if (_convert01 != null) return GuardNullable(_convert01); if (_convert10 != null) return GuardNullable(_convert10); throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Ivalid object state. If you see this message, please, report a bug."); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func Guard(Func convert) { return v => ClickHouseColumnReinterpreter.GuardNull(convert(v)); } private static Func Guard(Func convert) { return v => convert(v); } public static Func GuardNullable(Func convert) { return v => v == null ? (TRes?)null : ClickHouseColumnReinterpreter.GuardNull(convert(v)); } public static Func GuardNullable(Func convert) { return v => v == null ? (TRes?)null : convert(v); } public static Func GuardNullable(Func convert) { return v => v == null ? (TRes?)null : ClickHouseColumnReinterpreter.GuardNull(convert(v.Value)); } public static Func GuardNullable(Func convert) { return v => v == null ? (TRes?)null : convert(v.Value); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ClickHouseColumnSerializationMode.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { /// /// Provides supported serialization modes for /// and . /// public enum ClickHouseColumnSerializationMode { /// /// Default serialization. In this mode the reader should expect only column values. /// Default = 0, /// /// Sparse serialization. In this mode the reader should expect the list of 'granules' followed by column values. /// Sparse = 1, /// /// Custom serialization. In this mode the reader should expect serialization settings followed by actual column values. /// Custom = 0xAAAA } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ClickHouseEnumConverter.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Types { /// /// The class represents a converter that can convert values between a ClickHouse's enum and . /// The values of enums are matched by their codes (integers). /// /// The type of the enum. public sealed class ClickHouseEnumConverter : IClickHouseEnumConverter where TEnum : Enum { private readonly Dictionary _values; /// /// Initializes a new instance of the converter. 's values are acquired via reflection. /// public ClickHouseEnumConverter() { var enumValues = Enum.GetValues(typeof(TEnum)); _values = new Dictionary(enumValues.Length); foreach (var enumValue in enumValues) { var intValue = Convert.ToInt32(enumValue); _values[intValue] = (TEnum) enumValue!; } } T IClickHouseEnumConverter.Dispatch(IClickHouseEnumConverterDispatcher dispatcher) { return dispatcher.Dispatch(this); } /// /// Searches for a enum's value corresponding to a numeric value of the ClickHouse enum. /// /// The numeric value of the ClickHouse enum. /// The string value of the ClickHouse enum. This converter ignores the value of this parameter. /// When this method returns, contains the value of enum or the default value of when returns . /// if there are 's value corresponding to the specified ClickHouse enum; otherwise . public bool TryMap(int value, string stringValue, out TEnum enumValue) { return _values.TryGetValue(value, out enumValue!); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ClickHouseTableColumnHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal static class ClickHouseTableColumnHelper { public static Type? TryGetValueType(IClickHouseTableColumn column) { if (column.TryDipatch(ClickHouseTableColumnValueTypeDispatcher.Instance, out var type)) return type; return null; } private sealed class ClickHouseTableColumnValueTypeDispatcher : IClickHouseTableColumnDispatcher { public static readonly ClickHouseTableColumnValueTypeDispatcher Instance = new ClickHouseTableColumnValueTypeDispatcher(); private ClickHouseTableColumnValueTypeDispatcher() { } public Type Dispatch(IClickHouseTableColumn column) { return typeof(T); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ClickHouseTypeInfoProvider.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2022, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Net; using System.Numerics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { #region DefaultTypeInfoProvider (obsolete) /// /// This class is obsolete and will be deleted. Use instead of . /// [Obsolete(nameof(DefaultTypeInfoProvider) + " was renamed to " + nameof(ClickHouseTypeInfoProvider) + ".")] public class DefaultTypeInfoProvider : ClickHouseTypeInfoProvider { /// /// The instance of provides access to all types supported by ClickHouseClient. /// /// /// The class is obsolete. /// Use instead of . /// [Obsolete(nameof(DefaultTypeInfoProvider) + " was renamed to " + nameof(ClickHouseTypeInfoProvider) + ".")] public static readonly new DefaultTypeInfoProvider Instance = new DefaultTypeInfoProvider(); private DefaultTypeInfoProvider() : this(GetDefaultTypes()) { } /// /// Initializes a new instance of with a collection of supported types. /// /// The collection of supported types. /// /// The class is obsolete. /// Use the base class instead of . /// [Obsolete(nameof(DefaultTypeInfoProvider) + " was renamed to " + nameof(ClickHouseTypeInfoProvider) + ".")] protected DefaultTypeInfoProvider(IEnumerable types) : base(types) { } } #endregion DefaultTypeInfoProvider (obsolete) /// /// The default implementation of the interface . This class provides access to /// all types supported by ClickHouseClient. /// public class ClickHouseTypeInfoProvider : IClickHouseTypeInfoProvider { /// /// The instance of provides access to all types supported by ClickHouseClient. /// public static readonly ClickHouseTypeInfoProvider Instance = new ClickHouseTypeInfoProvider(); private readonly Dictionary _types; private ClickHouseTypeInfoProvider() : this(GetDefaultTypes()) { } /// /// Initializes a new instance of with a collection of supported types. /// /// The collection of supported types. /// It is possible to get types supported by default with the method . protected ClickHouseTypeInfoProvider(IEnumerable types) { if (types == null) throw new ArgumentNullException(nameof(types)); _types = types.ToDictionary(t => t.TypeName); } /// public IClickHouseColumnTypeInfo GetTypeInfo(string typeName) { var typeNameMem = typeName.AsMemory(); var (baseTypeName, options) = ParseTypeName(typeNameMem); var result = typeNameMem.Span == baseTypeName.Span ? GetTypeInfo(typeName, options) : GetTypeInfo(baseTypeName.ToString(), options); return result ?? throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeName}\" is not supported."); } /// public IClickHouseColumnTypeInfo GetTypeInfo(ReadOnlyMemory typeName) { var (baseTypeName, options) = ParseTypeName(typeName); var result = GetTypeInfo(baseTypeName.ToString(), options); return result ?? throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeName.ToString()}\" is not supported."); } private IClickHouseColumnTypeInfo? GetTypeInfo(string baseTypeName, List>? options) { if (!_types.TryGetValue(baseTypeName, out var typeInfo)) return null; if (options != null && options.Count > 0) typeInfo = typeInfo.GetDetailedTypeInfo(options, this); return typeInfo; } private static (ReadOnlyMemory baseTypeName, List>? options) ParseTypeName(ReadOnlyMemory typeName) { var typeNameSpan = typeName.Span; var pOpenIdx = typeNameSpan.IndexOf('('); if (pOpenIdx == 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The name of the type (\"{typeNameSpan.ToString()}\") can't start with \"(\"."); ReadOnlyMemory baseTypeName; List>? options = null; if (pOpenIdx < 0) { baseTypeName = typeName.Trim(); } else { baseTypeName = typeName.Slice(0, pOpenIdx).Trim(); int count = 1; int currentIdx = pOpenIdx; int optionStartIdx = pOpenIdx + 1; ReadOnlySpan significantChars = "(,)'`"; do { if (typeNameSpan.Length - 1 == currentIdx) break; var pNextIdx = typeNameSpan.Slice(currentIdx + 1).IndexOfAny(significantChars); if (pNextIdx < 0) break; pNextIdx += currentIdx + 1; currentIdx = pNextIdx; if ("'`".Contains(typeNameSpan[currentIdx])) { var len = ClickHouseSyntaxHelper.GetQuotedTokenLength(typeNameSpan.Slice(currentIdx), typeNameSpan[currentIdx]); if (len < 0) break; Debug.Assert(len > 0); currentIdx += len - 1; } else if (typeNameSpan[currentIdx] == '(') { ++count; } else if (typeNameSpan[currentIdx] == ')') { --count; if (count == 0) break; } else if (count == 1) { var currentOption = typeName.Slice(optionStartIdx, currentIdx - optionStartIdx).Trim(); optionStartIdx = currentIdx + 1; if (options != null) options.Add(currentOption); else options = new List>(2) {currentOption}; } } while (true); if (count != 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The number of open parentheses doesn't match to the number of close parentheses in the type name \"{typeNameSpan.ToString()}\"."); if (currentIdx != typeNameSpan.Length - 1) { var unexpectedString = typeNameSpan.Slice(currentIdx + 1); if (!unexpectedString.Trim().IsEmpty) { throw new ClickHouseException( ClickHouseErrorCodes.InvalidTypeName, $"There are unexpected characters (\"{unexpectedString.ToString()}\") in the type name \"{typeNameSpan.ToString()}\" after closing parenthesis."); } } var lastOption = typeName.Slice(optionStartIdx, currentIdx - optionStartIdx).Trim(); if (options != null) options.Add(lastOption); else options = new List>(1) {lastOption}; } return (baseTypeName, options); } /// public IClickHouseColumnTypeInfo GetTypeInfo(IClickHouseColumnTypeDescriptor typeDescriptor) { string? tzCode; string typeName; IntermediateClickHouseTypeInfo typeInfo; switch (typeDescriptor.ClickHouseDbType) { case ClickHouseDbType.AnsiString: case ClickHouseDbType.AnsiStringFixedLength: throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeDescriptor.ClickHouseDbType}\" is not supported. String encoding can be specified with the property \"{nameof(ClickHouseParameter.StringEncoding)}\"."); case ClickHouseDbType.Array: throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeDescriptor.ClickHouseDbType}\" is not supported. An array could be declared with properties \"{nameof(IClickHouseColumnDescriptor.ArrayRank)}\" or \"{nameof(ClickHouseParameter.IsArray)}\"."); case ClickHouseDbType.Enum: case ClickHouseDbType.Nothing: case ClickHouseDbType.Time: case ClickHouseDbType.Tuple: case ClickHouseDbType.Xml: case ClickHouseDbType.Map: throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeDescriptor.ClickHouseDbType}\" is not supported."); case ClickHouseDbType.Binary: if (typeDescriptor.Size <= 0) { typeInfo = new IntermediateClickHouseTypeInfo(ClickHouseDbType.Byte, "UInt8", false, 1); goto AFTER_TYPE_INFO_DEFINED; } typeName = string.Format(CultureInfo.InvariantCulture, "FixedString({0})", typeDescriptor.Size); break; case ClickHouseDbType.Byte: typeName = "UInt8"; break; case ClickHouseDbType.Boolean: typeName = "Bool"; break; case ClickHouseDbType.Currency: typeName = "Decimal(18, 4)"; break; case ClickHouseDbType.Date: typeName = "Date"; break; case ClickHouseDbType.Date32: typeName = "Date32"; break; case ClickHouseDbType.Decimal: typeName = string.Format(CultureInfo.InvariantCulture, "Decimal({0}, {1})", DecimalTypeInfoBase.DefaultPrecision, DecimalTypeInfoBase.DefaultScale); break; case ClickHouseDbType.Double: typeName = "Float64"; break; case ClickHouseDbType.Guid: typeName = "UUID"; break; case ClickHouseDbType.Int16: typeName = "Int16"; break; case ClickHouseDbType.Int32: typeName = "Int32"; break; case ClickHouseDbType.Int64: typeName = "Int64"; break; case ClickHouseDbType.Int128: typeName = "Int128"; break; case ClickHouseDbType.Int256: typeName = "Int256"; break; case ClickHouseDbType.Object: if (typeDescriptor.ValueType != typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeDescriptor.ClickHouseDbType}\" is not supported."); typeName = "Nothing"; break; case ClickHouseDbType.SByte: typeName = "Int8"; break; case ClickHouseDbType.Single: typeName = "Float32"; break; case ClickHouseDbType.String: typeName = "String"; break; case ClickHouseDbType.UInt16: typeName = "UInt16"; break; case ClickHouseDbType.UInt32: typeName = "UInt32"; break; case ClickHouseDbType.UInt64: typeName = "UInt64"; break; case ClickHouseDbType.UInt128: typeName = "UInt128"; break; case ClickHouseDbType.UInt256: typeName = "UInt256"; break; case ClickHouseDbType.VarNumeric: typeName = string.Format( CultureInfo.InvariantCulture, "Decimal({0}, {1})", typeDescriptor.Precision ?? DecimalTypeInfoBase.DefaultPrecision, typeDescriptor.Scale ?? DecimalTypeInfoBase.DefaultScale); break; case ClickHouseDbType.StringFixedLength: if (typeDescriptor.Size <= 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"The size of the fixed string must be a positive number."); typeName = string.Format(CultureInfo.InvariantCulture, "FixedString({0})", typeDescriptor.Size); break; case ClickHouseDbType.DateTime2: tzCode = GetTimeZoneCode(typeDescriptor.TimeZone); typeName = tzCode == null ? "DateTime64(7)" : $"DateTime64(7, '{tzCode}')"; break; case ClickHouseDbType.DateTime64: tzCode = GetTimeZoneCode(typeDescriptor.TimeZone); typeName = tzCode == null ? string.Format(CultureInfo.InvariantCulture, "DateTime64({0})", typeDescriptor.Precision ?? DateTime64TypeInfo.DefaultPrecision) : string.Format(CultureInfo.InvariantCulture, "DateTime64({0}, '{1}')", typeDescriptor.Precision ?? DateTime64TypeInfo.DefaultPrecision, tzCode); break; case ClickHouseDbType.DateTime: case ClickHouseDbType.DateTimeOffset: tzCode = GetTimeZoneCode(typeDescriptor.TimeZone); typeName = tzCode == null ? "DateTime" : $"DateTime('{tzCode}')"; break; case ClickHouseDbType.IpV4: typeName = "IPv4"; break; case ClickHouseDbType.IpV6: typeName = "IPv6"; break; case ClickHouseDbType.ClickHouseSpecificTypeDelimiterCode: goto default; case null: typeInfo = GetTypeFromValue(typeDescriptor.ValueType, typeDescriptor.IsNullable ?? false, typeDescriptor.TimeZone); goto AFTER_TYPE_INFO_DEFINED; default: throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"There is no type associated with the value \"{typeDescriptor.ClickHouseDbType}\"."); } if (typeDescriptor.IsNullable != null) { typeInfo = new IntermediateClickHouseTypeInfo(typeDescriptor.ClickHouseDbType.Value, typeName, typeDescriptor.IsNullable.Value, typeDescriptor.ArrayRank ?? 0); } else { // Derive nullability from the value's type. It's important to know whether the value is nullable or not because // nullability is a part of ClickHouse type var autoType = GetTypeFromValue(typeDescriptor.ValueType, typeDescriptor.IsNullable ?? false, typeDescriptor.TimeZone); typeInfo = new IntermediateClickHouseTypeInfo(typeDescriptor.ClickHouseDbType.Value, typeName, autoType.IsNullable, typeDescriptor.ArrayRank ?? 0); } // This label is an alternative exit point for switch // It's a shortcut for several cases when typeInfo is fully defined AFTER_TYPE_INFO_DEFINED: bool isNull = typeDescriptor.ValueType == typeof(DBNull); if (isNull && typeDescriptor.IsNullable == false) throw new ClickHouseException(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"The value of the type \"{typeDescriptor.ValueType}\" can't be declared as non-nullable."); bool isNullable; if (typeDescriptor.IsNullable != null) isNullable = typeDescriptor.IsNullable.Value; else if (isNull) isNullable = true; else isNullable = typeInfo.IsNullable; typeName = typeInfo.ClickHouseType; if (isNullable) typeName = $"Nullable({typeName})"; var arrayRank = typeDescriptor?.ArrayRank ?? typeInfo.ArrayRank; for (int i = 0; i < arrayRank; i++) typeName = $"Array({typeName})"; return GetTypeInfo(typeName); } internal static IntermediateClickHouseTypeInfo GetTypeFromValue(Type valueType, bool valueCanBeNull, TimeZoneInfo? timeZone) { if (valueType == typeof(string)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.String, "String", valueCanBeNull, 0); if (valueType == typeof(byte)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Byte, "UInt8", false, 0); if (valueType == typeof(bool)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Boolean, "Bool", false, 0); if (valueType == typeof(decimal)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Decimal, string.Format(CultureInfo.InvariantCulture, "Decimal({0}, {1})", DecimalTypeInfoBase.DefaultPrecision, DecimalTypeInfoBase.DefaultScale), false, 0); if (valueType == typeof(double)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Double, "Float64", false, 0); if (valueType == typeof(Guid)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Guid, "UUID", false, 0); if (valueType == typeof(short)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Int16, "Int16", false, 0); if (valueType == typeof(int)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Int32, "Int32", false, 0); if (valueType == typeof(long)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Int64, "Int64", false, 0); if (valueType == typeof(sbyte)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.SByte, "Int8", false, 0); if (valueType == typeof(BigInteger)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Int256, "Int256", false, 0); if (valueType == typeof(float)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Single, "Float32", false, 0); if (valueType == typeof(ushort)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.UInt16, "UInt16", false, 0); if (valueType == typeof(uint)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.UInt32, "UInt32", false, 0); if (valueType == typeof(ulong)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.UInt64, "UInt64", false, 0); if (valueType == typeof(IPAddress)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.IpV6, "IPv6", valueCanBeNull, 0); if (valueType == typeof(DBNull)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Nothing, "Nothing", true, 0); #if NET6_0_OR_GREATER if (valueType == typeof(DateOnly)) return new IntermediateClickHouseTypeInfo(ClickHouseDbType.Date, "Date", false, 0); #endif if (valueType == typeof(DateTime) || valueType == typeof(DateTimeOffset)) { var tzCode = GetTimeZoneCode(timeZone); return new IntermediateClickHouseTypeInfo(ClickHouseDbType.DateTime, tzCode == null ? "DateTime" : $"DateTime('{tzCode}')", false, 0); } int arrayRank = 1; Type? elementType = null; if (valueType.IsArray) { arrayRank = valueType.GetArrayRank(); elementType = valueType.GetElementType(); if (elementType == typeof(char)) { elementType = typeof(string); --arrayRank; } } else { foreach (var itf in valueType.GetInterfaces()) { if (!itf.IsGenericType) continue; if (itf.GetGenericTypeDefinition() != typeof(IReadOnlyList<>)) continue; var listElementType = itf.GetGenericArguments()[0]; if (elementType != null) { throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"The type \"{valueType}\" implements \"{typeof(IReadOnlyList<>)}\" at least twice with generic arguments \"{elementType}\" and \"{listElementType}\"."); } elementType = listElementType; } } if (elementType == null && valueType.IsGenericType) { var valueTypeDef = valueType.GetGenericTypeDefinition(); if (valueTypeDef == typeof(Memory<>) || valueTypeDef == typeof(ReadOnlyMemory<>)) { elementType = valueType.GetGenericArguments()[0]; // Memory or ReadOnlyMemory should be interpreted as string if (elementType == typeof(char)) return GetTypeFromValue(typeof(string), false, timeZone); } } if (elementType != null) { try { var elementInfo = GetTypeFromValue(elementType, arrayRank > 0 || valueCanBeNull, timeZone); return new IntermediateClickHouseTypeInfo(elementInfo.DbType, elementInfo.ClickHouseType, elementInfo.IsNullable, elementInfo.ArrayRank + arrayRank); } catch (ClickHouseException ex) { if (ex.ErrorCode != ClickHouseErrorCodes.TypeNotSupported) throw; throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"The type \"{valueType}\" is not supported. See the inner exception for details.", ex); } } if (valueType.IsGenericType && valueType.GetGenericTypeDefinition() == typeof(Nullable<>)) { elementType = valueType.GetGenericArguments()[0]; try { var elementInfo = GetTypeFromValue(elementType, false, timeZone); return new IntermediateClickHouseTypeInfo(elementInfo.DbType, elementInfo.ClickHouseType, true, elementInfo.ArrayRank); } catch (ClickHouseException ex) { if (ex.ErrorCode != ClickHouseErrorCodes.TypeNotSupported) throw; throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"The type \"{valueType}\" is not supported. See the inner exception for details.", ex); } } throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{valueType}\" is not supported."); } [return: NotNullIfNotNull("timeZone")] private static string? GetTimeZoneCode(TimeZoneInfo? timeZone) { if (timeZone == null) return null; return TimeZoneHelper.GetTimeZoneId(timeZone); } /// public IClickHouseTypeInfoProvider Configure(ClickHouseServerInfo serverInfo) { if (serverInfo == null) throw new ArgumentNullException(nameof(serverInfo)); return new ClickHouseTypeInfoProvider(_types.Values.Select(t => (t as IClickHouseConfigurableTypeInfo)?.Configure(serverInfo) ?? t)); } /// /// Returns all types supported by the ClickHouseClient. /// /// All types supported by the ClickHouseClient. protected static IEnumerable GetDefaultTypes() { return new IClickHouseColumnTypeInfo[] { new ArrayTypeInfo(), new LowCardinalityTypeInfo(), new TupleTypeInfo(), new BoolTypeInfo(), new DateTypeInfo(), new Date32TypeInfo(), new DateTimeTypeInfo(), new DateTime64TypeInfo(), new DecimalTypeInfo(), new Decimal32TypeInfo(), new Decimal64TypeInfo(), new Decimal128TypeInfo(), new Float32TypeInfo(), new Float64TypeInfo(), new Int8TypeInfo(), new Int16TypeInfo(), new Int32TypeInfo(), new Int64TypeInfo(), new Int128TypeInfo(), new Int256TypeInfo(), new UInt8TypeInfo(), new UInt16TypeInfo(), new UInt32TypeInfo(), new UInt64TypeInfo(), new UInt128TypeInfo(), new UInt256TypeInfo(), new StringTypeInfo(), new FixedStringTypeInfo(), new UuidTypeInfo(), new NothingTypeInfo(), new NullableTypeInfo(), new IpV4TypeInfo(), new IpV6TypeInfo(), new Enum8TypeInfo(), new Enum16TypeInfo(), new MapTypeInfo(), new VariantTypeInfo(), }; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/CustomSerializationColumnReader.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; namespace Octonica.ClickHouseClient.Types { internal sealed class CustomSerializationColumnReader : IClickHouseColumnReader, IClickHouseTableColumnDispatcher { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _typeInfo; private ClickHouseColumnSerializationMode _mode; private bool _trailingDefaults; private int _sparseRowPostion; private List? _offsets; private int _expectedBaseRowCount = -1; private IClickHouseColumnReader? _baseReader; public CustomSerializationColumnReader(IClickHouseColumnTypeInfo typeInfo, int rowCount, ClickHouseColumnSerializationMode mode) { _typeInfo = typeInfo; _mode = mode; _rowCount = rowCount; } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { if (_baseReader == null) return _typeInfo.CreateColumnReader(0).EndRead(settings); var columnInfo = _baseReader.EndRead(settings); if (_mode == ClickHouseColumnSerializationMode.Sparse) { if (columnInfo.TryDipatch(this, out var dispatchedColumn)) return dispatchedColumn; throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Sparse column reader error. A type of the column was not dispatched."); } return columnInfo; } IClickHouseTableColumn IClickHouseTableColumnDispatcher.Dispatch(IClickHouseTableColumn column) { Debug.Assert(_offsets != null); return new SparseColumn(column, _rowCount, _offsets, _trailingDefaults); } public SequenceSize ReadNext(ReadOnlySequence sequence) { switch (_mode) { case ClickHouseColumnSerializationMode.Custom: if (sequence.IsEmpty) return SequenceSize.Empty; // The prefix consists of a single byte encoding the serialization mode var mode = (ClickHouseColumnSerializationMode)sequence.FirstSpan[0]; if (mode == ClickHouseColumnSerializationMode.Sparse || mode == ClickHouseColumnSerializationMode.Default) { _mode = mode; var result = ReadNext(sequence.Slice(1)); return result.AddBytes(1); } throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Expected one of serialization modes: sparse or default. Received value: {mode}."); case ClickHouseColumnSerializationMode.Default: _baseReader ??= _typeInfo.CreateColumnReader(_rowCount); return _baseReader.ReadNext(sequence); case ClickHouseColumnSerializationMode.Sparse: return ReadSparse(sequence); default: throw new InvalidOperationException($"Internal error. Unexpected column serialization mode: {_mode}."); } } private SequenceSize ReadSparse(ReadOnlySequence sequence) { if (_expectedBaseRowCount == 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (_expectedBaseRowCount > 0) { Debug.Assert(_offsets != null); Debug.Assert(_baseReader != null); var result = _baseReader.ReadNext(sequence); _expectedBaseRowCount -= result.Elements; if (_expectedBaseRowCount < 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (_expectedBaseRowCount == 0) { // Calculate the number of rows with default values int defaultRowsCount = _rowCount - _offsets.Count; if (!_trailingDefaults) { var lastNonDefaultIdx = _offsets.Count == 0 ? -1 : _offsets[^1]; defaultRowsCount -= _rowCount - (lastNonDefaultIdx + 1); } return result.AddElements(defaultRowsCount); } return result; } if (_offsets == null) { // The default ratio from the docs // https://clickhouse.com/docs/en/operations/settings/merge-tree-settings#ratio_of_defaults_for_sparse_serialization const double defaultRatioForSparseSerialization = 0.9375; int capacity = Math.Max(16, (int)(_rowCount * (1 - defaultRatioForSparseSerialization))); _offsets = new List(capacity); } const ulong endOfGranuleFlag = 1ul << 62; var seq = sequence; int totalBytes = 0; while (true) { if (!ClickHouseBinaryProtocolReader.TryRead7BitInteger(seq, out var group_size, out var bytesRead)) return new SequenceSize(totalBytes, 0); totalBytes += bytesRead; seq = seq.Slice(bytesRead); var endOfGranule = (group_size & endOfGranuleFlag) == endOfGranuleFlag; group_size &= ~endOfGranuleFlag; _sparseRowPostion += checked((int)group_size); if (_sparseRowPostion >= _rowCount) { _trailingDefaults = true; _sparseRowPostion = _rowCount; if (endOfGranule) break; // It may be ok, but non-final group with an overflow looks suspicious Debug.Fail("Unexpected group after the end of the column"); continue; } _offsets.Add(_sparseRowPostion++); if (endOfGranule) break; } Debug.Assert(_rowCount >= _sparseRowPostion); Debug.Assert(_expectedBaseRowCount == -1); _expectedBaseRowCount = _offsets.Count; if (!_trailingDefaults) _expectedBaseRowCount += _rowCount - _sparseRowPostion; _baseReader = _typeInfo.CreateColumnReader(_expectedBaseRowCount); if (_expectedBaseRowCount == 0) { // The column consists only of default values return new SequenceSize(totalBytes, _rowCount); } Debug.Assert(_expectedBaseRowCount > 0); var baseResult = ReadSparse(seq); return baseResult.AddBytes(totalBytes); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/CustomSerializationSkippingColumnReader.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using System; using System.Buffers; using System.Diagnostics; namespace Octonica.ClickHouseClient.Types { internal sealed class CustomSerializationSkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _typeInfo; private ClickHouseColumnSerializationMode _mode; private int _sparseRowPostion; private int _expectedBaseRowCount = -1; private int _valueRowCount; private IClickHouseColumnReaderBase? _baseReader; public CustomSerializationSkippingColumnReader(IClickHouseColumnTypeInfo typeInfo, int rowCount, ClickHouseColumnSerializationMode mode) { _typeInfo = typeInfo; _mode = mode; _rowCount = rowCount; } public SequenceSize ReadNext(ReadOnlySequence sequence) { switch (_mode) { case ClickHouseColumnSerializationMode.Custom: if (sequence.IsEmpty) return SequenceSize.Empty; // The prefix consists of a single byte encoding the serialization mode var mode = (ClickHouseColumnSerializationMode)sequence.FirstSpan[0]; if (mode == ClickHouseColumnSerializationMode.Sparse || mode == ClickHouseColumnSerializationMode.Default) { _mode = mode; var result = ReadNext(sequence.Slice(1)); return result.AddBytes(1); } throw new ClickHouseException(ClickHouseErrorCodes.ProtocolUnexpectedResponse, $"Expected one of serialization modes: sparse or default. Received value: {mode}."); case ClickHouseColumnSerializationMode.Default: _baseReader ??= _typeInfo.CreateSkippingColumnReader(_rowCount); return _baseReader.ReadNext(sequence); case ClickHouseColumnSerializationMode.Sparse: return SkipSparse(sequence); default: throw new InvalidOperationException($"Internal error. Unexpected column serialization mode: {_mode}."); } } private SequenceSize SkipSparse(ReadOnlySequence sequence) { if (_expectedBaseRowCount == 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (_expectedBaseRowCount > 0) { Debug.Assert(_baseReader != null); var result = _baseReader.ReadNext(sequence); _expectedBaseRowCount -= result.Elements; if (_expectedBaseRowCount < 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (_expectedBaseRowCount == 0) { // Calculate the number of rows with default values int defaultRowsCount = _rowCount - _valueRowCount; return result.AddElements(defaultRowsCount); } return result; } const ulong endOfGranuleFlag = 1ul << 62; var seq = sequence; int totalBytes = 0; while (true) { if (!ClickHouseBinaryProtocolReader.TryRead7BitInteger(seq, out var group_size, out var bytesRead)) return new SequenceSize(totalBytes, 0); totalBytes += bytesRead; seq = seq.Slice(bytesRead); var endOfGranule = (group_size & endOfGranuleFlag) == endOfGranuleFlag; group_size &= ~endOfGranuleFlag; _sparseRowPostion += checked((int)group_size); if (_sparseRowPostion >= _rowCount) { _sparseRowPostion = _rowCount; if (endOfGranule) break; // It may be ok, but non-final group with an overflow looks suspicious Debug.Fail("Unexpected group after the end of the column"); continue; } ++_sparseRowPostion; ++_valueRowCount; if (endOfGranule) { _valueRowCount += _rowCount - _sparseRowPostion; break; } } Debug.Assert(_rowCount >= _sparseRowPostion); Debug.Assert(_expectedBaseRowCount == -1); _expectedBaseRowCount = _valueRowCount; _baseReader = _typeInfo.CreateSkippingColumnReader(_expectedBaseRowCount); if (_expectedBaseRowCount == 0) { // The column consists only of default values return new SequenceSize(totalBytes, _rowCount); } Debug.Assert(_expectedBaseRowCount > 0); var baseResult = ReadNext(seq); return baseResult.AddBytes(totalBytes); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TableColumn.Net6.0.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET6_0_OR_GREATER using System; namespace Octonica.ClickHouseClient.Types { partial class Date32TableColumn : IClickHouseTableColumn { private static readonly DateOnly UnixEpoch = DateOnly.FromDateTime(DateTime.UnixEpoch); // The default sparse value '1970-01-01' is not equal to the default column value '1901-01-01'. // Here we should return the value corresponding to 0, which is 1970-01-01. public DateOnly DefaultValue => UnixEpoch; public DateOnly GetValue(int index) { var value = _buffer.Span[index]; return UnixEpoch.AddDays(value); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(DateTime)) return (IClickHouseTableColumn)(object)new ReinterpretedTableColumn(this, dateOnly => dateOnly.ToDateTime(TimeOnly.MinValue)); if (typeof(T) == typeof(DateTime?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, dateOnly => dateOnly.ToDateTime(TimeOnly.MinValue))); if (typeof(T) == typeof(DateOnly?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, this); return null; } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TableColumn.NetCoreApp3.1.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NETCOREAPP3_1_OR_GREATER && !NET6_0_OR_GREATER using System; namespace Octonica.ClickHouseClient.Types { partial class Date32TableColumn : IClickHouseTableColumn { static readonly DateTime UnixEpochUnspecified = new DateTime(DateTime.UnixEpoch.Ticks, DateTimeKind.Unspecified); public DateTime DefaultValue => UnixEpochUnspecified; public DateTime GetValue(int index) { var value = _buffer.Span[index]; return UnixEpochUnspecified.AddDays(value); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(DateTime?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, this); return null; } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed partial class Date32TableColumn { private readonly ReadOnlyMemory _buffer; public int RowCount => _buffer.Length; public Date32TableColumn(ReadOnlyMemory buffer) { _buffer = buffer; } public bool IsNull(int index) { return false; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TypeInfo.Net6.0.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET6_0_OR_GREATER using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; namespace Octonica.ClickHouseClient.Types { partial class Date32TypeInfo { private static readonly DateOnly UnixEpoch = DateOnly.FromDateTime(DateTime.UnixEpoch); private static readonly DateOnly MinDateOnlyValue = UnixEpoch.AddDays(MinValue); private static readonly DateOnly MaxDateOnlyValue = UnixEpoch.AddDays(MaxValue); public override Type GetFieldType() { return typeof(DateOnly); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { IReadOnlyList dateOnlyRows; if (typeof(T) == typeof(DateOnly)) dateOnlyRows = (IReadOnlyList)rows; else if (typeof(T) == typeof(DateTime)) dateOnlyRows = ((IReadOnlyList)rows).Map(DateOnly.FromDateTime); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Date32Writer(columnName, ComplexTypeName, dateOnlyRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { DateOnly => new DateOnlyParameterWriter(this), DateTime => new DateTimeParameterWriter(this), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return (IClickHouseParameterWriter)writer; } private static int DateOnlyToDays(DateOnly value) { if (value == default) return MinValue; var days = value.DayNumber - UnixEpoch.DayNumber; if (days < MinValue || days > MaxValue) throw new OverflowException("The value must be in range [1925-01-01, 2283-11-11]."); return days; } partial class Date32Reader : StructureReaderBase { protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Date32TableColumn(buffer); } } partial class Date32Writer : StructureWriterBase { public Date32Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(int), rows) { } protected override int Convert(DateOnly value) { return DateOnlyToDays(value); } } private sealed class DateOnlyParameterWriter : IClickHouseParameterWriter { private readonly Date32TypeInfo _typeInfo; public DateOnlyParameterWriter(Date32TypeInfo typeInfo) { _typeInfo = typeInfo; } public bool TryCreateParameterValueWriter(DateOnly value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = ValueToString(value); if (isNested) strVal = $"'{strVal}'"; valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, DateOnly value) { var strVal = ValueToString(value); return queryBuilder.Append('\'').Append(strVal).Append("'::").Append(_typeInfo.ComplexTypeName); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string ValueToString(DateOnly value) { if (value == default) return DefaultValueStr; if (value < MinDateOnlyValue || value > MaxDateOnlyValue) throw new OverflowException($"The value must be in range [{MinDateOnlyValue}, {MaxDateOnlyValue}]."); return value.ToString(FormatStr, CultureInfo.InvariantCulture); } } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TypeInfo.NetCoreApp3.1.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NETCOREAPP3_1_OR_GREATER && !NET6_0_OR_GREATER using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Types { partial class Date32TypeInfo { public override Type GetFieldType() { return typeof(DateTime); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) != typeof(DateTime)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Date32Writer(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); if (type == typeof(DateTime)) return (IClickHouseParameterWriter)(object)new DateTimeParameterWriter(this); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } private static int DateTimeToDays(DateTime value) { if (value == default) return MinValue; var days = (value.Date - DateTime.UnixEpoch).TotalDays; if (days < MinValue || days > MaxValue) throw new OverflowException($"The value must be in range [{MinDateTimeValue}, {MaxDateTimeValue}]."); return (int)days; } partial class Date32Reader : StructureReaderBase { protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Date32TableColumn(buffer); } } partial class Date32Writer : StructureWriterBase { public Date32Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(int), rows) { } protected override int Convert(DateTime value) { return DateTimeToDays(value); } } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/Date32TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed partial class Date32TypeInfo : SimpleTypeInfo { public const int MinValue = -25567; public const int MaxValue = 120529; private const string FormatStr = DateTypeInfo.FormatStr; private const string DefaultValueStr = "1900-01-01"; private static readonly DateTime MinDateTimeValue = DateTime.UnixEpoch.AddDays(MinValue); private static readonly DateTime MaxDateTimeValue = DateTime.UnixEpoch.AddDays(MaxValue + 1).Subtract(TimeSpan.FromTicks(1)); public Date32TypeInfo() :base("Date32") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Date32Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(uint), rowCount); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Date32; } private sealed partial class Date32Reader { protected override bool BitwiseCopyAllowed => true; public Date32Reader(int rowCount) : base(sizeof(int), rowCount) { } protected override int ReadElement(ReadOnlySpan source) { return BitConverter.ToInt32(source); } } private sealed partial class Date32Writer { } private sealed class DateTimeParameterWriter : IClickHouseParameterWriter { private readonly Date32TypeInfo _typeInfo; public DateTimeParameterWriter(Date32TypeInfo typeInfo) { _typeInfo = typeInfo; } public bool TryCreateParameterValueWriter(DateTime value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = ValueToString(value); valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, DateTime value) { var strVal = ValueToString(value); return queryBuilder.Append('\'').Append(strVal).Append("\'::").Append(_typeInfo.ComplexTypeName); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string ValueToString(DateTime value) { if (value == default) return DefaultValueStr; if (value < MinDateTimeValue || value > MaxDateTimeValue) throw new OverflowException($"The value must be in range [{MinDateTimeValue}, {MaxDateTimeValue}]."); return value.ToString(FormatStr, CultureInfo.InvariantCulture); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateOnlyTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET6_0_OR_GREATER using System; namespace Octonica.ClickHouseClient.Types { internal sealed class DateOnlyTableColumn : StructureTableColumn { public DateOnlyTableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(DateTime)) return (IClickHouseTableColumn)(object)new ReinterpretedTableColumn(this, dateOnly => dateOnly.ToDateTime(TimeOnly.MinValue)); if (typeof(T) == typeof(DateTime?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, dateOnly => dateOnly.ToDateTime(TimeOnly.MinValue))); return base.TryReinterpret(); } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTime64TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class DateTime64TableColumn : IClickHouseTableColumn { private static readonly (long min, long max)[] ClickHouseTicksRange; private readonly ReadOnlyMemory _buffer; private readonly int _ticksScale; private readonly (long min, long max) _range; private readonly TimeZoneInfo _timeZone; public int RowCount { get; } public DateTimeOffset DefaultValue => new DateTimeOffset(DateTime.UnixEpoch).ToOffset(_timeZone.GetUtcOffset(DateTime.UnixEpoch)); static DateTime64TableColumn() { var ranges = new (long min, long max)[DateTime64TypeInfo.DateTimeTicksScales.Length]; var dateTimeTicksMax = (DateTime.MaxValue - DateTime.UnixEpoch).Ticks; var dateTimeTicksMin = (DateTime.MinValue - DateTime.UnixEpoch).Ticks; for (int i = 0; i < ranges.Length; i++) { long min, max; var magnitude = DateTime64TypeInfo.DateTimeTicksScales[i]; if (magnitude < 0) { if (dateTimeTicksMax > long.MaxValue / -magnitude) max = long.MaxValue; else max = checked(dateTimeTicksMax * -magnitude); if (dateTimeTicksMin < long.MinValue / -magnitude) min = long.MinValue; else min = checked(dateTimeTicksMin * -magnitude); } else { max = dateTimeTicksMax / magnitude; min = dateTimeTicksMin / magnitude; } ranges[i] = (min, max); } ClickHouseTicksRange = ranges; } public DateTime64TableColumn(ReadOnlyMemory buffer, int precision, TimeZoneInfo timeZone) { _buffer = buffer; _ticksScale = DateTime64TypeInfo.DateTimeTicksScales[precision]; _range = ClickHouseTicksRange[precision]; _timeZone = timeZone; RowCount = buffer.Length; } public bool IsNull(int index) { return false; } public DateTimeOffset GetValue(int index) { var ticks = _buffer.Span[index]; if (ticks < _range.min) { throw new OverflowException( $"The value 0x{ticks:X} is lesser than the minimal value of the type \"{typeof(DateTime)}\". " + $"It is only possible to read this value as \"{typeof(long)}\"."); } if (ticks > _range.max) { throw new OverflowException( $"The value 0x{ticks:X} is greater than the maximal value of the type \"{typeof(DateTime)}\". " + $"It is only possible to read this value as \"{typeof(long)}\"."); } if (_ticksScale < 0) ticks /= -_ticksScale; else ticks = checked(ticks * _ticksScale); var dateTime = DateTime.UnixEpoch.AddTicks(ticks); var offset = _timeZone.GetUtcOffset(dateTime); return new DateTimeOffset(dateTime).ToOffset(offset); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(DateTime)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, dto => dto.DateTime); if (typeof(T) == typeof(DateTime?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, dto => dto.DateTime)); if (typeof(T) == typeof(DateTimeOffset?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, this); var rawColumn = new StructureTableColumn(_buffer); var rawReinterpreted = rawColumn as IClickHouseTableColumn ?? rawColumn.TryReinterpret(); if (rawReinterpreted != null) return new ReinterpretedTableColumn(this, rawReinterpreted); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTime64TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class DateTime64TypeInfo : IClickHouseConfigurableTypeInfo { internal static readonly int[] DateTimeTicksScales; private static readonly (long min, long max)[] DateTimeTicksRanges; public const int DefaultPrecision = 3; private readonly int? _precision; private readonly string? _timeZoneCode; /// /// Indicates that the was acquired from the name of the type. /// private readonly bool _explicitTimeZoneCode; private TimeZoneInfo? _timeZone; public string ComplexTypeName { get; } public string TypeName => "DateTime64"; public int GenericArgumentsCount => 0; public int TypeArgumentsCount => (_precision == null ? 0 : 1) + (_timeZoneCode == null || !_explicitTimeZoneCode ? 0 : 1); static DateTime64TypeInfo() { const int maxPrecision = 9; var scales = new int[maxPrecision + 1]; for (int i = 0; i < scales.Length; i++) scales[i] = i == 0 ? 1 : checked(scales[i - 1] * 10); var ranges = new (long min, long max)[scales.Length]; var minValue = new DateTime(1900, 1, 1); var exclusiveMaxValue = new DateTime(2300, 1, 1); var dateTimeTicksMin = (minValue - DateTime.UnixEpoch).Ticks; var dateTimeTicksMax = (exclusiveMaxValue - DateTime.UnixEpoch).Ticks - 1; for (int i = 0; i < scales.Length; i++) { long scale, rem; if (scales[i] <= TimeSpan.TicksPerSecond) scale = Math.DivRem(TimeSpan.TicksPerSecond, scales[i], out rem); else scale = -Math.DivRem(scales[i], TimeSpan.TicksPerSecond, out rem); if (rem != 0) throw new InvalidOperationException($"Internal error. Expected that the value of {typeof(TimeSpan)}.{nameof(TimeSpan.TicksPerSecond)} is a power of 10."); scales[i] = checked((int)scale); long max; if (scale < 0) max = Math.Min(dateTimeTicksMax, long.MaxValue / -scale); else max = dateTimeTicksMax; long min; if (scale < 0) min = Math.Max(dateTimeTicksMin, long.MinValue / -scale); else min = dateTimeTicksMin; ranges[i] = (min, max); } DateTimeTicksScales = scales; DateTimeTicksRanges = ranges; } public DateTime64TypeInfo() : this(null, null, false) { } private DateTime64TypeInfo(int? precision, string? timeZoneCode, bool explicitTimeZoneCode) { if (precision != null) { if (precision.Value < 0) throw new ArgumentOutOfRangeException(nameof(precision), "The precision must be a non-negative number."); if (precision.Value >= DateTimeTicksScales.Length) throw new ArgumentOutOfRangeException(nameof(precision), $"The precision can't be greater than {DateTimeTicksScales.Length - 1}."); } _explicitTimeZoneCode = explicitTimeZoneCode; _precision = precision; _timeZoneCode = timeZoneCode; if (timeZoneCode != null && _explicitTimeZoneCode) { if (precision == null) throw new ArgumentNullException(nameof(precision)); ComplexTypeName = string.Format(CultureInfo.InvariantCulture, "{0}({1}, '{2}')", TypeName, precision.Value, timeZoneCode); } else if (precision != null) { ComplexTypeName = string.Format(CultureInfo.InvariantCulture, "{0}({1})", TypeName, precision.Value); } else { ComplexTypeName = TypeName; } } public Type GetFieldType() { return typeof(DateTimeOffset); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.DateTime64; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } public object GetTypeArgument(int index) { if (_precision == null) { Debug.Assert(_timeZoneCode == null || !_explicitTimeZoneCode); throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); } switch (index) { case 0: return _precision; case 1: if (_timeZoneCode != null && _explicitTimeZoneCode) return _timeZoneCode; goto default; default: throw new IndexOutOfRangeException(); } } public IClickHouseColumnReader CreateColumnReader(int rowCount) { var timeZone = GetTimeZone(); return new DateTime64Reader(rowCount, _precision ?? DefaultPrecision, timeZone); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(ulong), rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) == typeof(DateTime)) { var timeZone = GetTimeZone(); return new DateTimeWriter(columnName, ComplexTypeName, _precision ?? DefaultPrecision, timeZone, (IReadOnlyList)rows); } if (typeof(T) == typeof(DateTimeOffset)) { var timeZone = GetTimeZone(); return new DateTimeOffsetWriter(columnName, ComplexTypeName, _precision ?? DefaultPrecision, timeZone, (IReadOnlyList)rows); } if (typeof(T) == typeof(ulong)) return new UInt64TypeInfo.UInt64Writer(columnName, ComplexTypeName, (IReadOnlyList)rows); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object? converter = default(T) switch { ulong _ => null, DateTime _ => new DateTimeWriter("dummy", ComplexTypeName, _precision ?? DefaultPrecision, GetTimeZone(), Array.Empty()), DateTimeOffset _ => new DateTimeOffsetWriter("dummy", ComplexTypeName, _precision ?? DefaultPrecision, GetTimeZone(), Array.Empty()), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return new DateTime64ParameterWriter(this, (IConverter?)converter); } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (options.Count > 2) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); if (!int.TryParse(options[0].Span, NumberStyles.Integer, CultureInfo.InvariantCulture, out var precision)) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The first argument (precision) of the type \"{TypeName}\" must be an integer."); else if (precision < 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The precision value for the type \"{TypeName}\" must be a non-negative number."); if (options.Count == 2) { var tzCode = options[1].Trim('\'').ToString(); return new DateTime64TypeInfo(precision, tzCode, true); } return new DateTime64TypeInfo(precision, _timeZoneCode, false); } public IClickHouseColumnTypeInfo Configure(ClickHouseServerInfo serverInfo) { return new DateTime64TypeInfo(null, serverInfo.Timezone, false); } private TimeZoneInfo GetTimeZone() { if (_timeZone != null) return _timeZone; if (_timeZoneCode == null) return TimeZoneInfo.Utc; _timeZone = TimeZoneHelper.GetTimeZoneInfo(_timeZoneCode); return _timeZone; } private sealed class DateTime64Reader : StructureReaderBase { private readonly int _precision; private readonly TimeZoneInfo _timeZone; protected override bool BitwiseCopyAllowed => true; public DateTime64Reader(int rowCount, int precision, TimeZoneInfo timeZone) : base(sizeof(ulong), rowCount) { _precision = precision; _timeZone = timeZone; } protected override long ReadElement(ReadOnlySpan source) { return BitConverter.ToInt64(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new DateTime64TableColumn(buffer, _precision, _timeZone); } } private sealed class DateTimeWriter : StructureWriterBase, IConverter { private readonly int _ticksScale; private readonly TimeZoneInfo _timeZone; private readonly (long min, long max) _range; public DateTimeWriter(string columnName, string columnType, int precision, TimeZoneInfo timeZone, IReadOnlyList rows) : base(columnName, columnType, sizeof(ulong), rows) { _ticksScale = DateTimeTicksScales[precision]; _range = DateTimeTicksRanges[precision]; _timeZone = timeZone; } long IConverter.Convert(DateTime value) => Convert(value); protected override long Convert(DateTime value) { long ticks; if (value == default) { ticks = 0; } else { var dateTimeTicks = value.Ticks - DateTime.UnixEpoch.Ticks - _timeZone.GetUtcOffset(value).Ticks; if (dateTimeTicks < _range.min || dateTimeTicks > _range.max) throw new OverflowException($"The value must be in range [{DateTime.UnixEpoch.AddTicks(_range.min):O}, {DateTime.UnixEpoch.AddTicks(_range.max):O}]."); ticks = ScaleTicks(dateTimeTicks, _ticksScale); } return ticks; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static long ScaleTicks(long ticks, int scaleFactor) { if (scaleFactor < 0) return checked(ticks * -scaleFactor); if (ticks >= 0) return ticks / scaleFactor; var result = Math.DivRem(ticks, scaleFactor, out var rem); if (rem != 0) --result; return result; } } private sealed class DateTimeOffsetWriter : StructureWriterBase, IConverter { private readonly int _ticksScale; private readonly (long min, long max) _range; private readonly TimeZoneInfo _timeZone; public DateTimeOffsetWriter(string columnName, string columnType, int precision, TimeZoneInfo timeZone, IReadOnlyList rows) : base(columnName, columnType, sizeof(ulong), rows) { _ticksScale = DateTimeTicksScales[precision]; _range = DateTimeTicksRanges[precision]; _timeZone = timeZone; } long IConverter.Convert(DateTimeOffset value) => Convert(value); protected override long Convert(DateTimeOffset value) { long ticks; if (value == default) { ticks = 0; } else { var offset = _timeZone.GetUtcOffset(value); var valueWithOffset = value.ToOffset(offset); var dateTimeTicks = (valueWithOffset - DateTimeOffset.UnixEpoch).Ticks; if (dateTimeTicks < _range.min || dateTimeTicks > _range.max) throw new OverflowException($"The value must be in range [{DateTimeOffset.UnixEpoch.AddTicks(_range.min):O}, {DateTimeOffset.UnixEpoch.AddTicks(_range.max):O}]."); ticks = DateTimeWriter.ScaleTicks(dateTimeTicks, _ticksScale); } return ticks; } } private sealed class DateTime64ParameterWriter : IClickHouseParameterWriter { private readonly DateTime64TypeInfo _typeInfo; private readonly IConverter? _converter; public DateTime64ParameterWriter(DateTime64TypeInfo typeInfo, IConverter? converter) { Debug.Assert(converter != null || typeof(T) == typeof(long)); _typeInfo = typeInfo; _converter = converter; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var ticks = Convert(value); var str = ticks.ToString(CultureInfo.InvariantCulture); valueWriter = new SimpleLiteralValueWriter(str.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { var ticks = Convert(value); queryBuilder.Append("reinterpret(cast("); queryBuilder.Append(ticks.ToString(CultureInfo.InvariantCulture)); queryBuilder.Append(",'Int64'),'"); queryBuilder.Append(_typeInfo.ComplexTypeName.Replace("'", "''")); queryBuilder.Append("')"); return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { var ticksType = typeInfoProvider.GetTypeInfo("Int64"); return writeValue(queryBuilder, ticksType, (qb, realWrite) => { qb.Append("reinterpret("); realWrite(queryBuilder); qb.Append(",'"); qb.Append(_typeInfo.ComplexTypeName.Replace("'", "''")); qb.Append("')"); return qb; }); } private long Convert(T value) { if (_converter == null) { Debug.Assert(typeof(T) == typeof(long)); Debug.Assert(value != null); return (long)(object)value; } return _converter.Convert(value); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTimeTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class DateTimeTableColumn : IClickHouseTableColumn { private readonly ReadOnlyMemory _buffer; private readonly TimeZoneInfo _timeZone; public int RowCount { get; } public DateTimeOffset DefaultValue => default; public DateTimeTableColumn(ReadOnlyMemory buffer, TimeZoneInfo timeZone) { _buffer = buffer; _timeZone = timeZone; RowCount = buffer.Length; } public bool IsNull(int index) { return false; } public DateTimeOffset GetValue(int index) { var seconds = _buffer.Span[index]; if (seconds == 0) return default; var dateTime = DateTime.UnixEpoch.AddSeconds(seconds); var offset = _timeZone.GetUtcOffset(dateTime); return new DateTimeOffset(dateTime).ToOffset(offset); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(DateTime)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, dto => dto.DateTime); if (typeof(T) == typeof(DateTime?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, dto => dto.DateTime)); if (typeof(T) == typeof(DateTimeOffset?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, this); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTimeTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class DateTimeTypeInfo : IClickHouseConfigurableTypeInfo { private readonly string? _timeZoneCode; /// /// Indicates that the was acquired from the name of the type. /// private readonly bool _explicitTimeZoneCode; private TimeZoneInfo? _timeZone; public string ComplexTypeName { get; } public string TypeName => "DateTime"; public int GenericArgumentsCount => 0; public int TypeArgumentsCount => _timeZoneCode == null || !_explicitTimeZoneCode ? 0 : 1; public DateTimeTypeInfo() : this(null, false) { } private DateTimeTypeInfo(string? timeZoneCode, bool explicitTimeZoneCode) { _timeZoneCode = timeZoneCode; _explicitTimeZoneCode = explicitTimeZoneCode; ComplexTypeName = timeZoneCode == null || !_explicitTimeZoneCode ? TypeName : $"{TypeName}('{timeZoneCode}')"; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { var timeZone = GetTimeZone(); return new DateTimeReader(rowCount, timeZone); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(uint), rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) == typeof(DateTime)) { var timeZone = GetTimeZone(); return new DateTimeWriter(columnName, ComplexTypeName, timeZone, (IReadOnlyList)rows); } if (typeof(T) == typeof(DateTimeOffset)) { var timeZone = GetTimeZone(); return new DateTimeOffsetWriter(columnName, ComplexTypeName, timeZone, (IReadOnlyList)rows); } throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object converter = default(T) switch { DateTime _ => new DateTimeWriter("dummy", ComplexTypeName, GetTimeZone(), Array.Empty()), DateTimeOffset _ => new DateTimeOffsetWriter("dummy", ComplexTypeName, GetTimeZone(), Array.Empty()), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return new DateTimeParameterWriter(this, (IConverter)converter); } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (options.Count > 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); var tzCode = options[0].Trim('\'').ToString(); return new DateTimeTypeInfo(tzCode, true); } public Type GetFieldType() { return typeof(DateTimeOffset); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.DateTimeOffset; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } public object GetTypeArgument(int index) { if (_timeZoneCode == null || !_explicitTimeZoneCode) throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); if (index != 0) throw new IndexOutOfRangeException(); return _timeZoneCode; } public IClickHouseColumnTypeInfo Configure(ClickHouseServerInfo serverInfo) { return new DateTimeTypeInfo(serverInfo.Timezone, false); } private TimeZoneInfo GetTimeZone() { if (_timeZone != null) return _timeZone; if (_timeZoneCode == null) return TimeZoneInfo.Utc; _timeZone = TimeZoneHelper.GetTimeZoneInfo(_timeZoneCode); return _timeZone; } private sealed class DateTimeReader : StructureReaderBase { private readonly TimeZoneInfo _timeZone; protected override bool BitwiseCopyAllowed => true; public DateTimeReader(int rowCount, TimeZoneInfo timeZone) : base(sizeof(uint), rowCount) { _timeZone = timeZone; } protected override uint ReadElement(ReadOnlySpan source) { return BitConverter.ToUInt32(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new DateTimeTableColumn(buffer, _timeZone); } } private sealed class DateTimeWriter : StructureWriterBase, IConverter { private readonly TimeZoneInfo _timeZone; public DateTimeWriter(string columnName, string columnType, TimeZoneInfo timeZone, IReadOnlyList rows) : base(columnName, columnType, sizeof(uint), rows) { _timeZone = timeZone; } uint IConverter.Convert(DateTime value) { return Convert(value); } protected override uint Convert(DateTime value) { uint seconds; if (value == default) { seconds = 0; } else { var doubleSeconds = (value - DateTime.UnixEpoch).TotalSeconds - _timeZone.GetUtcOffset(value).TotalSeconds; if (doubleSeconds < 0 || doubleSeconds > uint.MaxValue) throw new OverflowException("The value must be in range [1970-01-01 00:00:00, 2105-12-31 23:59:59]."); seconds = (uint) doubleSeconds; } return seconds; } } private sealed class DateTimeOffsetWriter : StructureWriterBase, IConverter { private readonly TimeZoneInfo _timeZone; public DateTimeOffsetWriter(string columnName, string columnType, TimeZoneInfo timeZone, IReadOnlyList rows) : base(columnName, columnType, sizeof(uint), rows) { _timeZone = timeZone; } uint IConverter.Convert(DateTimeOffset value) { return Convert(value); } protected override uint Convert(DateTimeOffset value) { uint seconds; if (value == default) { seconds = 0; } else { var offset = _timeZone.GetUtcOffset(value); var valueWithOffset = value.ToOffset(offset); var doubleSeconds = (valueWithOffset - DateTimeOffset.UnixEpoch).TotalSeconds; if (doubleSeconds < 0 || doubleSeconds > uint.MaxValue) throw new OverflowException("The value must be in range [1970-01-01 00:00:00, 2105-12-31 23:59:59]."); seconds = (uint)doubleSeconds; } return seconds; } } private sealed class DateTimeParameterWriter : IClickHouseParameterWriter { private readonly DateTimeTypeInfo _typeInfo; private readonly IConverter _converter; public DateTimeParameterWriter(DateTimeTypeInfo typeInfo, IConverter converter) { _typeInfo = typeInfo; _converter = converter; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var seconds = _converter.Convert(value); var str = seconds.ToString(CultureInfo.InvariantCulture); valueWriter = new SimpleLiteralValueWriter(str.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { queryBuilder.Append("CAST("); var seconds = _converter.Convert(value); queryBuilder.Append(seconds.ToString(CultureInfo.InvariantCulture)); return queryBuilder.AppendFormat(" AS ").Append(_typeInfo.ComplexTypeName).Append(')'); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { var ticksType = typeInfoProvider.GetTypeInfo("UInt32"); return writeValue(queryBuilder, ticksType, (qb, realWrite) => { qb.Append("CAST("); realWrite(qb); return qb.AppendFormat(" AS ").Append(_typeInfo.ComplexTypeName).Append(')'); }); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTypeInfo.Net6.0.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET6_0_OR_GREATER using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; namespace Octonica.ClickHouseClient.Types { partial class DateTypeInfo { private static readonly DateOnly UnixEpoch = DateOnly.FromDateTime(DateTime.UnixEpoch); private static readonly DateOnly MaxDateOnlyValue = UnixEpoch.AddDays(ushort.MaxValue); public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { IReadOnlyList dateOnlyRows; if (typeof(T) == typeof(DateOnly)) dateOnlyRows = (IReadOnlyList)rows; else if (typeof(T) == typeof(DateTime)) dateOnlyRows = ((IReadOnlyList)rows).Map(DateOnly.FromDateTime); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new DateWriter(columnName, ComplexTypeName, dateOnlyRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { DateOnly => new DateOnlyParameterWriter(this), DateTime => new DateTimeParameterWriter(this), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(DateOnly); } private static ushort DateOnlyToDays(DateOnly value) { if (value == default) return 0; var days = value.DayNumber - UnixEpoch.DayNumber; if (days < 0 || days > ushort.MaxValue) throw new OverflowException("The value must be in range [1970-01-01, 2149-06-06]."); return (ushort)days; } partial class DateReader : StructureReaderBase { public DateReader(int rowCount) : base(sizeof(ushort), rowCount) { } protected override DateOnly ReadElement(ReadOnlySpan source) { var value = BitConverter.ToUInt16(source); if (value == 0) return default; return UnixEpoch.AddDays(value); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new DateOnlyTableColumn(buffer); } } partial class DateWriter : StructureWriterBase { public DateWriter(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(ushort), rows) { } protected override ushort Convert(DateOnly value) { return DateOnlyToDays(value); } } private sealed class DateOnlyParameterWriter : IClickHouseParameterWriter { private readonly DateTypeInfo _typeInfo; public DateOnlyParameterWriter(DateTypeInfo typeInfo) { _typeInfo = typeInfo; } public bool TryCreateParameterValueWriter(DateOnly value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = ValueToString(value); if (isNested) strVal = $"'{strVal}'"; valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, DateOnly value) { var strVal = ValueToString(value); return queryBuilder.Append('\'').Append(strVal).Append("'::").Append(_typeInfo.ComplexTypeName); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string ValueToString(DateOnly value) { if (value == default) return DefaultValueStr; if (value < UnixEpoch || value > MaxDateOnlyValue) throw new OverflowException($"The value must be in range [{DateTime.UnixEpoch}, {MaxDateOnlyValue}]."); return value.ToString(FormatStr, CultureInfo.InvariantCulture); } } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTypeInfo.NetCoreApp3.1.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NETCOREAPP3_1_OR_GREATER && !NET6_0_OR_GREATER using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Types { partial class DateTypeInfo { public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) != typeof(DateTime)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new DateWriter(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); if (type == typeof(DateTime)) return (IClickHouseParameterWriter)(object)new DateTimeParameterWriter(this); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } private static ushort DateTimeToDays(DateTime value) { if (value == default) return 0; var days = (value - DateTime.UnixEpoch).TotalDays; if (days < 0 || days > ushort.MaxValue) throw new OverflowException("The value must be in range [1970-01-01, 2149-06-06]."); return (ushort)days; } public override Type GetFieldType() { return typeof(DateTime); } partial class DateReader : StructureReaderBase { static readonly DateTime UnixEpochUnspecified = new DateTime(DateTime.UnixEpoch.Ticks, DateTimeKind.Unspecified); public DateReader(int rowCount) : base(sizeof(ushort), rowCount) { } protected override DateTime ReadElement(ReadOnlySpan source) { var value = BitConverter.ToUInt16(source); if (value == 0) return default; return UnixEpochUnspecified.AddDays(value); } } partial class DateWriter : StructureWriterBase { public DateWriter(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(ushort), rows) { } protected override ushort Convert(DateTime value) { return DateTimeToDays(value); } } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Types/DateTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed partial class DateTypeInfo : SimpleTypeInfo { public const string FormatStr = "yyyy-MM-dd"; private const string DefaultValueStr = "1970-01-01"; private static readonly DateTime MaxDateTimeValue = DateTime.UnixEpoch.AddDays(ushort.MaxValue); public DateTypeInfo() : base("Date") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new DateReader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(ushort), rowCount); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Date; } private sealed partial class DateReader { } private sealed partial class DateWriter { } private sealed class DateTimeParameterWriter : IClickHouseParameterWriter { private readonly DateTypeInfo _typeInfo; public DateTimeParameterWriter(DateTypeInfo typeInfo) { _typeInfo = typeInfo; } public bool TryCreateParameterValueWriter(DateTime value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = ValueToString(value); if (isNested) strVal = $"'{strVal}'"; valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, DateTime value) { var strVal = ValueToString(value); return queryBuilder.Append('\'').Append(strVal).Append("'::").Append(_typeInfo.ComplexTypeName); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static string ValueToString(DateTime value) { if (value == default) return DefaultValueStr; if (value < DateTime.UnixEpoch || value > MaxDateTimeValue) throw new OverflowException($"The value must be in range [{DateTime.UnixEpoch}, {MaxDateTimeValue}]."); return value.ToString(FormatStr, CultureInfo.InvariantCulture); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Decimal128TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Decimal128TypeInfo : DecimalTypeInfoBase { private const int Precision = 38; public override int TypeArgumentsCount => Math.Min(1, base.TypeArgumentsCount); public Decimal128TypeInfo() : base("Decimal128") { } private Decimal128TypeInfo(string typeName, string complexTypeName, int scale) : base(typeName, complexTypeName, Precision, scale) { } protected override DecimalTypeInfoBase CloneWithOptions(string complexTypeName, int? precision, int scale) { if (precision != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The value of the precision can not be redefined for the type \"{TypeName}\"."); return new Decimal128TypeInfo(TypeName, complexTypeName, scale); } public override object GetTypeArgument(int index) { if (base.TypeArgumentsCount == 0) return base.GetTypeArgument(index); if (index != 0) throw new IndexOutOfRangeException(); return base.GetTypeArgument(1); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Decimal32TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Decimal32TypeInfo : DecimalTypeInfoBase { private const int Precision = 9; public override int TypeArgumentsCount => Math.Min(1, base.TypeArgumentsCount); public Decimal32TypeInfo() : base("Decimal32") { } private Decimal32TypeInfo(string typeName, string complexTypeName, int scale) : base(typeName, complexTypeName, Precision, scale) { } protected override DecimalTypeInfoBase CloneWithOptions(string complexTypeName, int? precision, int scale) { if (precision != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The value of the precision can not be redefined for the type \"{TypeName}\"."); return new Decimal32TypeInfo(TypeName, complexTypeName, scale); } public override object GetTypeArgument(int index) { if (base.TypeArgumentsCount == 0) return base.GetTypeArgument(index); if (index != 0) throw new IndexOutOfRangeException(); return base.GetTypeArgument(1); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Decimal64TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Decimal64TypeInfo : DecimalTypeInfoBase { private const int Precision = 18; public override int TypeArgumentsCount => Math.Min(1, base.TypeArgumentsCount); public Decimal64TypeInfo() : base("Decimal64") { } private Decimal64TypeInfo(string typeName, string complexTypeName, int scale) : base(typeName, complexTypeName, Precision, scale) { } protected override DecimalTypeInfoBase CloneWithOptions(string complexTypeName, int? precision, int scale) { if (precision != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The value of the precision can not be redefined for the type \"{TypeName}\"."); return new Decimal64TypeInfo(TypeName, complexTypeName, scale); } public override object GetTypeArgument(int index) { if (base.TypeArgumentsCount == 0) return base.GetTypeArgument(index); if (index != 0) throw new IndexOutOfRangeException(); return base.GetTypeArgument(1); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DecimalTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class DecimalTableColumn : IClickHouseTableColumn { private const int MaxDecimalScale = 28; private static readonly uint[] Scales = {10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000, 1_000_000_000}; private readonly ReadOnlyMemory _buffer; private readonly int _elementSize; private readonly byte _scale; public int RowCount => _buffer.Length / _elementSize; public decimal DefaultValue => decimal.Zero; public DecimalTableColumn(ReadOnlyMemory buffer, int elementSize, byte scale) { if (elementSize != 1 && elementSize != 2 && elementSize != 4) throw new ArgumentOutOfRangeException(nameof(elementSize)); _buffer = buffer; _elementSize = elementSize; _scale = scale; } public bool IsNull(int index) { return false; } public decimal GetValue(int index) { var startIndex = index * _elementSize; var span = _buffer.Span; var lowLow = span[startIndex]; bool isNegative; uint lowHigh, highLow; if (_elementSize == 1) { isNegative = (lowLow & unchecked((uint) int.MinValue)) != 0; if (isNegative) lowLow = unchecked(0 - lowLow); lowHigh = highLow = 0; } else if (_elementSize == 2) { lowHigh = span[startIndex + 1]; isNegative = (lowHigh & unchecked((uint) int.MinValue)) != 0; if (isNegative) { if (lowLow == 0) { lowHigh = unchecked(0 - lowHigh); } else { lowLow = unchecked(0 - lowLow); lowHigh ^= uint.MaxValue; } } highLow = 0; } else { lowHigh = span[startIndex + 1]; highLow = span[startIndex + 2]; var highHigh = span[startIndex + 3]; isNegative = (highHigh & unchecked((uint) int.MinValue)) != 0; if (isNegative) { if (lowLow == 0) { if (lowHigh == 0) { if (highLow == 0) { highHigh = unchecked(0 - highHigh); } else { highLow = unchecked(0 - highLow); highHigh ^= uint.MaxValue; } } else { lowHigh = unchecked(0 - lowHigh); highLow ^= uint.MaxValue; highHigh ^= uint.MaxValue; } } else { lowLow = unchecked(0 - lowLow); lowHigh ^= uint.MaxValue; highLow ^= uint.MaxValue; highHigh ^= uint.MaxValue; } } if (highHigh != 0 || _scale > MaxDecimalScale) { uint mll = lowLow, mlh = lowHigh, mhl = highLow, mhh = highHigh; byte scale = _scale; var deltaScale = _scale > MaxDecimalScale ? _scale - MaxDecimalScale : 0; // Attempt to rescale the value without loss of significant digits. It should work for values written by this DB driver. while (mhh != 0 || deltaScale != 0) { int scaleIndex = 0; if (deltaScale > 0) { scaleIndex = Math.Min(deltaScale, Scales.Length) - 1; deltaScale -= (byte) (scaleIndex + 1); } else { for (; scaleIndex < Scales.Length - 1; scaleIndex++) { if (mhh < Scales[scaleIndex]) break; } } var mul = Scales[scaleIndex]; ulong rem = mhh % mul; mhh /= mul; var val = mhl | rem << 32; rem = val % mul; mhl = (uint) (val / mul); val = mlh | rem << 32; rem = val % mul; mlh = (uint) (val / mul); val = mll | rem << 32; rem = val % mul; if (rem != 0 || scaleIndex >= scale) throw new NotSupportedException($"Value 0x{highHigh:x8}_{highLow:x8}_{lowHigh:x8}_{lowLow:x8} is too long for the type \"{typeof(decimal).FullName}\"."); mll = (uint) (val / mul); scale = (byte) (scale - scaleIndex - 1); } return new decimal(unchecked((int) mll), unchecked((int) mlh), unchecked((int) mhl), isNegative, scale); } } return new decimal(unchecked((int) lowLow), unchecked((int) lowHigh), unchecked((int) highLow), isNegative, _scale); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(decimal?)) return (IClickHouseTableColumn)(object)new NullableStructTableColumn(null, this); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DecimalTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Types { internal sealed class DecimalTypeInfo : DecimalTypeInfoBase { public DecimalTypeInfo() : base("Decimal") { } private DecimalTypeInfo(string typeName, string complexTypeName, int precision, int scale) : base(typeName, complexTypeName, precision, scale) { } protected override DecimalTypeInfoBase CloneWithOptions(string complexTypeName, int? precision, int scale) { if (precision == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The precision is required for the type \"{TypeName}\"."); return new DecimalTypeInfo(TypeName, complexTypeName, precision.Value, scale); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/DecimalTypeInfoBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.InteropServices; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal abstract class DecimalTypeInfoBase : IClickHouseColumnTypeInfo { public const byte DefaultPrecision = 38, DefaultScale = 9; private readonly int? _precision; private readonly int? _scale; public string ComplexTypeName { get; } public string TypeName { get; } public int GenericArgumentsCount => 0; public virtual int TypeArgumentsCount => (_precision == null ? 0 : 1) + (_scale == null ? 0 : 1); protected DecimalTypeInfoBase(string typeName) { TypeName = typeName; ComplexTypeName = typeName; } protected DecimalTypeInfoBase(string typeName, string complexTypeName, int precision, int scale) { if (precision < 1 || precision > 38) throw new ArgumentOutOfRangeException(nameof(precision), "The precision must be in the range [1:38]."); if (scale < 0) throw new ArgumentOutOfRangeException(nameof(scale), "The scale must be a non-negative number."); if (scale > precision) throw new ArgumentOutOfRangeException(nameof(scale), "The scale must not be greater than the precision."); TypeName = typeName; ComplexTypeName = complexTypeName; _precision = precision; _scale = scale; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_precision == null || _scale == null) { if (_precision == null && _scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Both scale and precision are required for the type \"{TypeName}\"."); if (_scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Scale is required for the type \"{TypeName}\"."); // Currently there is no implementation which requires only the precision value throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Precision is required for the type \"{TypeName}\"."); } return new DecimalReader(_precision.Value, _scale.Value, rowCount); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_precision == null || _scale == null) { if (_precision == null && _scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Both scale and precision are required for the type \"{TypeName}\"."); if (_scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Scale is required for the type \"{TypeName}\"."); // Currently there is no implementation which requires only the precision value throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Precision is required for the type \"{TypeName}\"."); } return new SimpleSkippingColumnReader(GetElementSize(_precision.Value), rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_precision == null && _scale == null) { var specifiedType = CloneWithOptions(string.Format(CultureInfo.InvariantCulture, "Decimal128({0})", DefaultScale), DefaultPrecision, DefaultScale); return specifiedType.CreateColumnWriter(columnName, rows, columnSettings); } if (_scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Scale is required for the type \"{TypeName}\"."); if (_precision == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Precision is required for the type \"{TypeName}\"."); var type = typeof(T); IReadOnlyList decimalRows; if (type == typeof(decimal)) decimalRows = (IReadOnlyList)rows; else if (type == typeof(long)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(ulong)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(int)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(uint)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(short)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(ushort)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(sbyte)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) decimalRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new DecimalWriter(columnName, ComplexTypeName, _precision.Value, _scale.Value, decimalRows); } public IClickHouseParameterWriter CreateParameterWriter() { if (_precision == null && _scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Both scale and precision are required for the type \"{TypeName}\"."); if (_scale == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Scale is required for the type \"{TypeName}\"."); if (_precision == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"Precision is required for the type \"{TypeName}\"."); var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); var binaryWriter = new DecimalWriter("Value", ComplexTypeName, _precision.Value, _scale.Value, Array.Empty()); object writer = default(T) switch { decimal _ => new DecimalParameterWriter(this, binaryWriter, v => v), long _ => new DecimalParameterWriter(this, binaryWriter, v => v), ulong _ => new DecimalParameterWriter(this, binaryWriter, v => v), int _ => new DecimalParameterWriter(this, binaryWriter, v => v), uint _ => new DecimalParameterWriter(this, binaryWriter, v => v), short _ => new DecimalParameterWriter(this, binaryWriter, v => v), ushort _ => new DecimalParameterWriter(this, binaryWriter, v => v), sbyte _ => new DecimalParameterWriter(this, binaryWriter, v => v), byte _ => new DecimalParameterWriter(this, binaryWriter, v => v), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."), }; return (IClickHouseParameterWriter)writer; } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { int? precision = null; int scale; if (options.Count == 1) { if (!int.TryParse(options[0].Span, NumberStyles.Integer, CultureInfo.InvariantCulture, out scale) || scale < 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The scale value for the type \"{TypeName}\" must be a non-negative number."); } else if (options.Count == 2) { if (!int.TryParse(options[0].Span, NumberStyles.Integer, CultureInfo.InvariantCulture, out var firstValue) || firstValue <= 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The first parameter in options (precision) for the type \"{TypeName}\" must be a positive number."); precision = firstValue; if (!int.TryParse(options[1].Span, NumberStyles.Integer, CultureInfo.InvariantCulture, out scale) || scale < 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The second parameter in options (scale) for the type \"{TypeName}\" must be a non-negative number."); } else { throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"Too many options for the type \"{TypeName}\"."); } if (_precision != null && precision != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The value of the precision can not be redefined for the type \"{TypeName}\"."); var complexTypeName = TypeName + "(" + string.Join(", ", options) + ")"; return CloneWithOptions(complexTypeName, precision, scale); } public Type GetFieldType() { return typeof(decimal); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Decimal; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } public virtual object GetTypeArgument(int index) { if (_precision == null && _scale == null) throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); switch (index) { case 0: if (_precision == null) { Debug.Assert(_scale != null); return _scale; } else { return _precision; } case 1: if (_scale == null || _precision == null) goto default; return _scale; default: throw new IndexOutOfRangeException(); } } protected abstract DecimalTypeInfoBase CloneWithOptions(string complexTypeName, int? precision, int scale); private static int GetElementSize(int precision) { if (precision <= 9) return 4; if (precision <= 18) return 8; Debug.Assert(precision <= 38); return 16; } private sealed class DecimalReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly int _elementSize; private readonly byte _scale; private readonly uint[] _values; private int _position; public DecimalReader(int precision, int scale, int rowCount) { _rowCount = rowCount; _elementSize = GetElementSize(precision); _values = new uint[_elementSize / 4 * rowCount]; _scale = (byte) scale; } public SequenceSize ReadNext(ReadOnlySequence sequence) { var elementPosition = _position * sizeof(uint) / _elementSize; if (elementPosition >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var byteLength = (int) Math.Min((_rowCount - elementPosition) * _elementSize, sequence.Length - sequence.Length % _elementSize); var uintLength = byteLength / sizeof(uint); var targetSpan = MemoryMarshal.AsBytes(new Span(_values, _position, uintLength)); Debug.Assert(targetSpan.Length == byteLength); sequence.Slice(0, byteLength).CopyTo(targetSpan); _position += uintLength; return new SequenceSize(byteLength, byteLength / _elementSize); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return EndReadInternal(); } private DecimalTableColumn EndReadInternal() { var memory = new ReadOnlyMemory(_values, 0, _position); return new DecimalTableColumn(memory, _elementSize / 4, _scale); } } private sealed class DecimalWriter : StructureWriterBase { private const byte MaxDecimalScale = 28; private static readonly uint[] Scales = { 10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000, 100_000_000, 1_000_000_000 }; private readonly byte _scale; public new int ElementSize { get => base.ElementSize; } public DecimalWriter(string columnName, string columnType, int precision, int scale, IReadOnlyList rows) : base(columnName, columnType, GetElementSize(precision), rows) { _scale = (byte) scale; } public void WriteDecimal(Span writeTo, decimal value) { WriteElement(writeTo, value); } protected override void WriteElement(Span writeTo, in decimal value) { var rescaledValue = Math.Round(value, Math.Min(_scale, MaxDecimalScale), MidpointRounding.AwayFromZero); #if NET5_0_OR_GREATER Span bits = stackalloc int[4]; decimal.GetBits(rescaledValue, bits); #else var bits = decimal.GetBits(rescaledValue); #endif uint lowLow = unchecked((uint) bits[0]); uint lowHigh = unchecked((uint) bits[1]); uint highLow = unchecked((uint) bits[2]); uint highHigh = 0; bool isNegative = (bits[3] & int.MinValue) != 0; int scale = (bits[3] & ~int.MinValue) >> 16; var deltaScale = _scale - scale; if (deltaScale < 0) throw new InvalidOperationException("Internal error: unexpected scale difference."); bool overflow = false; while (deltaScale > 0) { var iterationScale = Math.Min(deltaScale, Scales.Length); deltaScale -= iterationScale; var multiplier = (ulong) Scales[iterationScale - 1]; ulong lowLowMul = lowLow * multiplier; ulong lowHighMul = lowHigh * multiplier; ulong highLowMul = highLow * multiplier; ulong highHighMul = highHigh * multiplier; lowLow = unchecked((uint) lowLowMul); lowHigh = unchecked((uint) lowHighMul); highLow = unchecked((uint) highLowMul); highHigh = unchecked((uint) highHighMul); var val = lowLowMul >> 32; if (val != 0) { val += lowHigh; lowHigh = unchecked((uint) val); val >>= 32; if (val != 0) { val += highLow; highLow = unchecked((uint) val); val >>= 32; if (val != 0) { val += highHigh; highHigh = unchecked((uint) val); val >>= 32; if (val != 0) { overflow = true; break; } } } } val = lowHighMul >> 32; if (val != 0) { val += highLow; highLow = unchecked((uint)val); val >>= 32; if (val != 0) { val += highHigh; highHigh = unchecked((uint)val); val >>= 32; if (val != 0) { overflow = true; break; } } } val = highLowMul >> 32; if (val != 0) { val += highHigh; highHigh = unchecked((uint)val); val >>= 32; if (val != 0) { overflow = true; break; } } val = highHighMul >> 32; if (val != 0) { overflow = true; break; } } if (!overflow) { if (isNegative) { lowLow = unchecked(0 - lowLow); uint max = lowLow == 0 ? 0 : uint.MaxValue; lowHigh = unchecked(max - lowHigh); if (lowHigh != 0 && max == 0) max = uint.MaxValue; highLow = unchecked(max - highLow); if (highLow != 0 && max == 0) max = uint.MaxValue; highHigh = unchecked(max - highHigh); if (ElementSize == 4) overflow = highHigh != uint.MaxValue || highLow != uint.MaxValue || lowHigh != uint.MaxValue || (lowLow & unchecked((uint) int.MinValue)) == 0; else if (ElementSize == 8) overflow = highHigh != uint.MaxValue || highLow != uint.MaxValue || (lowHigh & unchecked((uint) int.MinValue)) == 0; else overflow = (highHigh & unchecked((uint) int.MinValue)) == 0; if (overflow && rescaledValue == 0) overflow = false; } else { if (ElementSize == 4) overflow = highHigh != 0 || highLow != 0 || lowHigh != 0 || (lowLow & unchecked((uint) int.MinValue)) != 0; else if (ElementSize == 8) overflow = highHigh != 0 || highLow != 0 || (lowHigh & unchecked((uint) int.MinValue)) != 0; else overflow = (highHigh & unchecked((uint) int.MinValue)) != 0; } } if (overflow) throw new OverflowException($"The decimal value is too big and can't be written to the column of type \"{ColumnType}\"."); var success = BitConverter.TryWriteBytes(writeTo, lowLow); Debug.Assert(success); if (ElementSize == 4) return; success = BitConverter.TryWriteBytes(writeTo.Slice(4), lowHigh); Debug.Assert(success); if (ElementSize == 8) return; Debug.Assert(ElementSize == 16); success = BitConverter.TryWriteBytes(writeTo.Slice(8), highLow); Debug.Assert(success); success = BitConverter.TryWriteBytes(writeTo.Slice(12), highHigh); Debug.Assert(success); } } private sealed class DecimalParameterWriter : IClickHouseParameterWriter { private readonly DecimalTypeInfoBase _type; private readonly DecimalWriter _binaryWriter; private readonly Func _convert; public DecimalParameterWriter(DecimalTypeInfoBase type, DecimalWriter binaryWriter, Func convert) { _type = type; _binaryWriter = binaryWriter; _convert = convert; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { Memory binaryValue = new byte[_binaryWriter.ElementSize]; GetBytes(value, binaryValue.Span); int lastNonZeroIdx = 0; for (int i = 0; i < binaryValue.Length; i++) { if (binaryValue.Span[i] == 0) continue; lastNonZeroIdx = i; } binaryValue = binaryValue.Slice(0, lastNonZeroIdx + 1); valueWriter = new HexStringLiteralValueWriter(binaryValue, isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { Span buffer = stackalloc byte[_binaryWriter.ElementSize]; GetBytes(value, buffer); // Not all decimal values can be parsed: // > Real value ranges that can be stored in memory are a bit larger than specified above, which are checked only on conversion from a string. // But reinterpret_cast lets obtain any Decimal value queryBuilder.Append("reinterpret("); HexStringParameterWriter.Interpolate(queryBuilder, buffer); queryBuilder.Append(", '"); queryBuilder.Append(_binaryWriter.ColumnType); return queryBuilder.Append("')"); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { var typeName = $"FixedString({_binaryWriter.ElementSize.ToString(CultureInfo.InvariantCulture)})"; var type = typeInfoProvider.GetTypeInfo(typeName); return writeValue(queryBuilder, type, (sb, realWrite) => { sb.Append("reinterpret("); realWrite(sb); sb.Append(", '"); sb.Append(_binaryWriter.ColumnType); return queryBuilder.Append("')"); }); } private void GetBytes(T value, Span buffer) { var decValue = _convert(value); _binaryWriter.WriteDecimal(buffer, decValue); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/EmptyParameterValueWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using System; namespace Octonica.ClickHouseClient.Types { internal sealed class EmptyParameterValueWriter : IClickHouseParameterValueWriter { public static readonly EmptyParameterValueWriter Instance = new EmptyParameterValueWriter(); public int Length => 0; private EmptyParameterValueWriter() { } public int Write(Memory buffer) { return 0; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Enum16TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class Enum16TypeInfo : EnumTypeInfoBase { public Enum16TypeInfo() : base("Enum16") { } private Enum16TypeInfo(string typeName, string complexTypeName, IEnumerable> values) : base(typeName, complexTypeName, values) { } protected override EnumColumnReaderBase CreateColumnReader(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap) { return new EnumColumnReader(internalReader, reversedEnumMap); } protected override SimpleSkippingColumnReader CreateInternalSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(short), rowCount); } protected override IClickHouseColumnTypeInfo CreateDetailedTypeInfo(string complexTypeName, IEnumerable> values) { return new Enum16TypeInfo(TypeName, complexTypeName, values); } protected override StructureReaderBase CreateInternalColumnReader(int rowCount) { return new Int16TypeInfo.Int16Reader(rowCount); } protected override IClickHouseColumnWriter CreateInternalColumnWriter(string columnName, IReadOnlyList rows) { var type = typeof(T); IReadOnlyList shortRows; if (type == typeof(short)) shortRows = (IReadOnlyList)rows; else if (type == typeof(byte)) shortRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(sbyte)) shortRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{TypeName}\"."); return new Int16TypeInfo.Int16Writer(columnName, ComplexTypeName, shortRows); } public override IClickHouseParameterWriter CreateParameterWriter() { // TODO: ClickHouseDbType.Enum is not supported in DefaultTypeInfoProvider.GetTypeInfo if (_enumMap == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The list of items is not specified."); var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(string)) { writer = new EnumParameterWriter(this); } else { writer = default(T) switch { short _ => new SimpleParameterWriter(this), byte _ => new SimpleParameterWriter(this), sbyte _ => new SimpleParameterWriter(this), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."), }; } return (IClickHouseParameterWriter)writer; } protected override bool TryParse(ReadOnlySpan text, out short value) { return short.TryParse(text, out value); } private sealed class EnumColumnReader : EnumColumnReaderBase { public EnumColumnReader(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap) : base(internalReader, reversedEnumMap) { } protected override EnumTableColumnDispatcherBase CreateColumnDispatcher(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap) { return new EnumTableColumnDispatcher(column, reversedEnumMap); } } private sealed class EnumTableColumnDispatcher : EnumTableColumnDispatcherBase { public EnumTableColumnDispatcher(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap) : base(column, reversedEnumMap) { } protected override bool TryMap(IClickHouseEnumConverter enumConverter, short value, string stringValue, out TEnum enumValue) { return enumConverter.TryMap(value, stringValue, out enumValue); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Enum8TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal sealed class Enum8TypeInfo : EnumTypeInfoBase { public Enum8TypeInfo() : base("Enum8") { } private Enum8TypeInfo(string typeName, string complexTypeName, IEnumerable> values) : base(typeName, complexTypeName, values) { } protected override EnumColumnReaderBase CreateColumnReader(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap) { return new EnumColumnReader(internalReader, reversedEnumMap); } protected override IClickHouseColumnTypeInfo CreateDetailedTypeInfo(string complexTypeName, IEnumerable> values) { return new Enum8TypeInfo(TypeName, complexTypeName, values); } protected override StructureReaderBase CreateInternalColumnReader(int rowCount) { return new Int8TypeInfo.Int8Reader(rowCount); } protected override SimpleSkippingColumnReader CreateInternalSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(byte), rowCount); } protected override IClickHouseColumnWriter CreateInternalColumnWriter(string columnName, IReadOnlyList rows) { if (typeof(T) != typeof(sbyte)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{TypeName}\"."); return new Int8TypeInfo.Int8Writer(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { // TODO: ClickHouseDbType.Enum is not supported in DefaultTypeInfoProvider.GetTypeInfo if (_enumMap == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The list of items is not specified."); var type = typeof(T); if (typeof(T) == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(string)) writer = new EnumParameterWriter(this); else if (type == typeof(sbyte)) writer = new SimpleParameterWriter(this); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } protected override bool TryParse(ReadOnlySpan text, out sbyte value) { return sbyte.TryParse(text, out value); } private sealed class EnumColumnReader : EnumColumnReaderBase { public EnumColumnReader(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap) : base(internalReader, reversedEnumMap) { } protected override EnumTableColumnDispatcherBase CreateColumnDispatcher(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap) { return new EnumTableColumnDispatcher(column, reversedEnumMap); } } private sealed class EnumTableColumnDispatcher : EnumTableColumnDispatcherBase { public EnumTableColumnDispatcher(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap) : base(column, reversedEnumMap) { } protected override bool TryMap(IClickHouseEnumConverter enumConverter, sbyte value, string stringValue, out TEnum enumValue) { return enumConverter.TryMap(value, stringValue, out enumValue); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/EnumTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; namespace Octonica.ClickHouseClient.Types { internal sealed class EnumTableColumn : IClickHouseTableColumn where TKey : struct { private readonly IClickHouseTableColumn _internalColumn; private readonly IReadOnlyDictionary _valueMap; public int RowCount => _internalColumn.RowCount; public string DefaultValue { get; } public EnumTableColumn(IClickHouseTableColumn internalColumn, IReadOnlyDictionary valueMap) { _internalColumn = internalColumn; _valueMap = valueMap; if (!_valueMap.TryGetValue(_internalColumn.DefaultValue, out var defaultStr)) defaultStr = string.Empty; DefaultValue = defaultStr; } public bool IsNull(int index) { Debug.Assert(!_internalColumn.IsNull(index)); return false; } public string GetValue(int index) { var value = _internalColumn.GetValue(index); if (!_valueMap.TryGetValue(value, out var strValue)) throw new InvalidCastException($"There is no string representation for the value {value} in the enum."); return strValue; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { var internalReinterpreted = _internalColumn as IClickHouseTableColumn ?? _internalColumn.TryReinterpret(); if (internalReinterpreted != null) return new ReinterpretedTableColumn(this, internalReinterpreted); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } internal sealed class EnumTableColumn : IClickHouseTableColumn where TKey : struct where TEnum : Enum { private readonly IClickHouseTableColumn _internalColumn; private readonly IReadOnlyDictionary _enumMap; private readonly IReadOnlyDictionary _stringMap; public int RowCount => _internalColumn.RowCount; public TEnum DefaultValue { get; } public EnumTableColumn(IClickHouseTableColumn internalColumn, IReadOnlyDictionary enumMap, IReadOnlyDictionary stringMap) { _internalColumn = internalColumn; _enumMap = enumMap; _stringMap = stringMap; if (!_enumMap.TryGetValue(_internalColumn.DefaultValue, out var defaultEnum)) defaultEnum = default; Debug.Assert(defaultEnum != null); DefaultValue = defaultEnum; } public bool IsNull(int index) { Debug.Assert(!_internalColumn.IsNull(index)); return false; } public TEnum GetValue(int index) { var value = _internalColumn.GetValue(index); if (!_enumMap.TryGetValue(value, out var strValue)) { if (_stringMap.TryGetValue(value, out var nativeValue)) throw new InvalidCastException($"The value '{nativeValue}'={value} of the enum can't be converted to the type '{typeof(Enum).FullName}'."); throw new InvalidCastException($"The value {value} doesn't belong to the enum."); } return strValue; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { IClickHouseTableColumn? reinterpretedColumn; if (typeof(T) == typeof(string)) reinterpretedColumn = (IClickHouseTableColumn) (object) new EnumTableColumn(_internalColumn, _stringMap); else reinterpretedColumn = _internalColumn.TryReinterpret(); if (reinterpretedColumn == null) return null; return new ReinterpretedTableColumn(this, reinterpretedColumn); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/EnumTypeInfoBase.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal abstract class EnumTypeInfoBase : IClickHouseColumnTypeInfo where TValue : struct, IFormattable { protected readonly Dictionary? _enumMap; private readonly Dictionary? _reversedEnumMap; private readonly List? _mapOrder; public string ComplexTypeName { get; } public string TypeName { get; } public int GenericArgumentsCount => 0; public int TypeArgumentsCount => _mapOrder?.Count ?? 0; protected EnumTypeInfoBase(string typeName) { TypeName = typeName; ComplexTypeName = typeName; } protected EnumTypeInfoBase(string typeName, string complexTypeName, IEnumerable> values) { TypeName = typeName; ComplexTypeName = complexTypeName; _enumMap = new Dictionary(StringComparer.Ordinal); _reversedEnumMap = new Dictionary(); _mapOrder = new List(); foreach (var pair in values) { _enumMap.Add(pair.Key, pair.Value); _reversedEnumMap.Add(pair.Value, pair.Key); _mapOrder.Add(pair.Key); } } public Type GetFieldType() { return typeof(string); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Enum; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } public object GetTypeArgument(int index) { if (_mapOrder == null || _enumMap == null) throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); var key = _mapOrder[index]; var value = _enumMap[key]; return new KeyValuePair(key, value); } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_enumMap == null || _reversedEnumMap == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The list of items is not specified."); var internalReader = CreateInternalColumnReader(rowCount); return CreateColumnReader(internalReader, _reversedEnumMap); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_enumMap == null || _reversedEnumMap == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The list of items is not specified."); return CreateInternalSkippingColumnReader(rowCount); } protected abstract EnumColumnReaderBase CreateColumnReader(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap); public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_enumMap == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The list of items is not specified."); if (typeof(T) == typeof(string)) { var list = MappedReadOnlyList.Map( (IReadOnlyList)rows, key => key == null ? default : _enumMap.TryGetValue(key, out var value) ? value : throw new InvalidCastException($"The value \"{key}\" can't be converted to {ComplexTypeName}.")); return CreateInternalColumnWriter(columnName, list); } return CreateInternalColumnWriter(columnName, rows); } public abstract IClickHouseParameterWriter CreateParameterWriter(); public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { var parsedOptions = new List>(options.Count); var complexNameBuilder = new StringBuilder(TypeName).Append('('); bool isFirst = true; foreach (var option in options) { if (isFirst) isFirst = false; else complexNameBuilder.Append(", "); var keyStrLen = ClickHouseSyntaxHelper.GetSingleQuoteStringLength(option.Span); if (keyStrLen < 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The fragment \"{option}\" is not recognized as an item of the enum."); var key = ClickHouseSyntaxHelper.GetSingleQuoteString(option.Slice(0, keyStrLen).Span); var valuePart = option.Slice(keyStrLen); var eqSignIdx = valuePart.Span.IndexOf('='); if (eqSignIdx < 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The fragment \"{option}\" is not recognized as an item of the enum."); valuePart = valuePart.Slice(eqSignIdx + 1).Trim(); if (!TryParse(valuePart.Span, out var value)) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The value {valuePart} is not a valid value of {TypeName}."); complexNameBuilder.Append(option); parsedOptions.Add(new KeyValuePair(key, value)); } var complexName = complexNameBuilder.Append(')').ToString(); return CreateDetailedTypeInfo(complexName, parsedOptions); } protected abstract IClickHouseColumnTypeInfo CreateDetailedTypeInfo(string complexTypeName, IEnumerable> values); protected abstract StructureReaderBase CreateInternalColumnReader(int rowCount); protected abstract SimpleSkippingColumnReader CreateInternalSkippingColumnReader(int rowCount); protected abstract IClickHouseColumnWriter CreateInternalColumnWriter(string columnName, IReadOnlyList rows); protected abstract bool TryParse(ReadOnlySpan text, out TValue value); protected abstract class EnumColumnReaderBase : IClickHouseColumnReader { private readonly StructureReaderBase _internalReader; private readonly IReadOnlyDictionary _reversedEnumMap; public EnumColumnReaderBase(StructureReaderBase internalReader, IReadOnlyDictionary reversedEnumMap) { _internalReader = internalReader; _reversedEnumMap = reversedEnumMap; } public SequenceSize ReadNext(ReadOnlySequence sequence) { return _internalReader.ReadNext(sequence); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { var column = _internalReader.EndRead(null); var enumConverter = settings?.EnumConverter; if (enumConverter != null) { var dispatcher = CreateColumnDispatcher(column, _reversedEnumMap); return enumConverter.Dispatch(dispatcher); } return new EnumTableColumn(column, _reversedEnumMap); } protected abstract EnumTableColumnDispatcherBase CreateColumnDispatcher(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap); } protected abstract class EnumTableColumnDispatcherBase : IClickHouseEnumConverterDispatcher { private readonly IClickHouseTableColumn _column; private readonly IReadOnlyDictionary _reversedEnumMap; public EnumTableColumnDispatcherBase(IClickHouseTableColumn column, IReadOnlyDictionary reversedEnumMap) { _column = column; _reversedEnumMap = reversedEnumMap; } public IClickHouseTableColumn Dispatch(IClickHouseEnumConverter enumConverter) where TEnum : Enum { var map = new Dictionary(_reversedEnumMap.Count); foreach (var pair in _reversedEnumMap) { if (TryMap(enumConverter, pair.Key, pair.Value, out var enumValue)) map.Add(pair.Key, enumValue); } return new EnumTableColumn(_column, map, _reversedEnumMap); } protected abstract bool TryMap(IClickHouseEnumConverter enumConverter, TValue value, string stringValue, out TEnum enumValue) where TEnum : Enum; } protected sealed class EnumParameterWriter : IClickHouseParameterWriter { private readonly EnumTypeInfoBase _type; private readonly SimpleParameterWriter _writer; public EnumParameterWriter(EnumTypeInfoBase type) { _type = type; _writer = new SimpleParameterWriter(_type); } public bool TryCreateParameterValueWriter(string value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var enumValue = Convert(value); return _writer.TryCreateParameterValueWriter(enumValue, isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, string value) { var enumValue = Convert(value); return _writer.Interpolate(queryBuilder, enumValue); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _writer.Interpolate(queryBuilder, typeInfoProvider, writeValue); } private TValue Convert(string value) { var enumMap = _type._enumMap; Debug.Assert(enumMap != null); if (enumMap.TryGetValue(value, out var enumValue)) return enumValue; throw new InvalidCastException($"The value \"{value}\" can't be converted to the ClickHouse type \"{_type.ComplexTypeName}\"."); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/FixedStringDecodedCharArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class FixedStringDecodedCharArrayTableColumn : FixedStringTableColumnBase { public override char[] DefaultValue => Array.Empty(); public FixedStringDecodedCharArrayTableColumn(Memory buffer, int rowSize, Encoding encoding) : base(buffer, rowSize, encoding) { } protected override char[] GetValue(Encoding encoding, ReadOnlySpan span) { var charCount = encoding.GetCharCount(span); var result = new char[charCount]; encoding.GetChars(span, result); int pos; for (pos = result.Length - 1; pos >= 0; pos--) { if (result[pos] != 0) break; } if (pos == 0) return Array.Empty(); if (pos + 1 < result.Length) Array.Resize(ref result, pos + 1); return result; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/FixedStringDecodedTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class FixedStringDecodedTableColumn : FixedStringTableColumnBase { public override string DefaultValue => string.Empty; public FixedStringDecodedTableColumn(Memory buffer, int rowSize, Encoding encoding) : base(buffer, rowSize, encoding) { } protected override string GetValue(Encoding encoding, ReadOnlySpan span) { var charCount = encoding.GetCharCount(span); var charSpan = new Span(new char[charCount]); encoding.GetChars(span, charSpan); int pos; for (pos = charSpan.Length - 1; pos >= 0; pos--) { if (charSpan[pos] != 0) break; } charSpan = charSpan.Slice(0, pos + 1); if (charSpan.IsEmpty) return string.Empty; return new string(charSpan); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/FixedStringTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class FixedStringTableColumn : FixedStringTableColumnBase { public override byte[] DefaultValue => Array.Empty(); public FixedStringTableColumn(Memory buffer, int rowSize, Encoding encoding) : base(buffer, rowSize, encoding) { } [return: NotNull] protected override byte[] GetValue(Encoding encoding, ReadOnlySpan span) { var result = new byte[span.Length]; span.CopyTo(result); return result; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/FixedStringTableColumnBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Octonica.ClickHouseClient.Types { internal abstract class FixedStringTableColumnBase : IClickHouseTableColumn, IClickHouseArrayTableColumn { private readonly Memory _buffer; private readonly int _rowSize; private readonly Encoding _encoding; public int RowCount { get; } public abstract TOut DefaultValue { get; } protected FixedStringTableColumnBase(Memory buffer, int rowSize, Encoding encoding) { _buffer = buffer; _rowSize = rowSize; _encoding = encoding; RowCount = _buffer.Length / _rowSize; } public bool IsNull(int index) { return false; } [return: NotNull] public TOut GetValue(int index) { return GetValue(_encoding, _buffer.Span.Slice(index * _rowSize, _rowSize)); } [return: NotNull] protected abstract TOut GetValue(Encoding encoding, ReadOnlySpan span); object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(string)) return (IClickHouseTableColumn)(object)new FixedStringDecodedTableColumn(_buffer, _rowSize, _encoding); if (typeof(T) == typeof(byte[])) return (IClickHouseTableColumn)(object)new FixedStringTableColumn(_buffer, _rowSize, _encoding); if (typeof(T) == typeof(char[])) return (IClickHouseTableColumn)(object)new FixedStringDecodedCharArrayTableColumn(_buffer, _rowSize, _encoding); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } public int CopyTo(int index, Span buffer, int dataOffset) { if (dataOffset < 0 || dataOffset > _rowSize) throw new ArgumentOutOfRangeException(nameof(dataOffset)); var length = Math.Min(_rowSize - dataOffset, buffer.Length); _buffer.Span.Slice(index * _rowSize + dataOffset, length).CopyTo(buffer); return length; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/FixedStringTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class FixedStringTypeInfo : IClickHouseColumnTypeInfo { private readonly int? _length; public string ComplexTypeName { get; } public string TypeName => "FixedString"; public int GenericArgumentsCount => 0; public int TypeArgumentsCount => _length == 0 ? 0 : 1; public FixedStringTypeInfo() { ComplexTypeName = TypeName; } private FixedStringTypeInfo(int length) { _length = length; ComplexTypeName = string.Format(CultureInfo.InvariantCulture, "{0}({1})", TypeName, length); } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_length == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The length of the fixed string is not specified."); return new FixedStringReader(rowCount, _length.Value); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_length == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The length of the fixed string is not specified."); return new SimpleSkippingColumnReader(_length.Value, rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_length == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The length of the fixed string is not specified."); var type = typeof(T); if (type == typeof(byte[])) return new FixedStringBytesColumnWriter(columnName, ComplexTypeName, (IReadOnlyList)rows, _length.Value); if (type == typeof(string)) return new FixedStringStringColumnWriter(columnName, ComplexTypeName, (IReadOnlyList)rows, _length.Value, columnSettings?.StringEncoding); if (type == typeof(ReadOnlyMemory)) return new FixedStringBytesColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>)rows, _length.Value); if (type == typeof(Memory)) return new FixedStringBytesColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>)rows, _length.Value); if (type == typeof(ReadOnlyMemory)) return new FixedStringStringColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>)rows, _length.Value, columnSettings?.StringEncoding); if (type == typeof(Memory)) return new FixedStringStringColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>)rows, _length.Value, columnSettings?.StringEncoding); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public IClickHouseParameterWriter CreateParameterWriter() { if (_length == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, "The length of the fixed string is not specified."); var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values"); object writer; if (type == typeof(string)) writer = new FixedStringParameterWriter(this, s => s.AsMemory()); else if (type == typeof(ReadOnlyMemory)) writer = new FixedStringParameterWriter(this); else if (type == typeof(Memory)) writer = new FixedStringParameterWriter>(this, mem => mem); else if (type == typeof(byte[])) writer = new FixedStringHexParameterWriter(this, a => a.AsMemory()); else if (type == typeof(ReadOnlyMemory)) writer = new FixedStringHexParameterWriter(this); else if (type == typeof(Memory)) writer = new FixedStringHexParameterWriter>(this, mem => mem); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (_length != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); if (options.Count > 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); if (!int.TryParse(options[0].Span, NumberStyles.Integer, CultureInfo.InvariantCulture, out var length) || length <= 0) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The length of \"{TypeName}({options[0].ToString()})\" must be a positive number."); return new FixedStringTypeInfo(length); } public Type GetFieldType() { return typeof(byte[]); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.StringFixedLength; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } public object GetTypeArgument(int index) { if (_length == null) throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); if (index == 0) return _length; throw new IndexOutOfRangeException(); } private sealed class FixedStringReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly int _rowSize; private readonly Memory _buffer; private int _position; public FixedStringReader(int rowCount, int rowSize) { _rowCount = rowCount; _rowSize = rowSize; if (rowCount > 0) _buffer = new Memory(new byte[rowCount * rowSize]); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position / _rowSize >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (sequence.Length < _rowSize) return new SequenceSize(0, 0); var elementsCount = Math.Min((int) sequence.Length / _rowSize, _rowCount - _position / _rowSize); if (elementsCount == 0) return new SequenceSize(0, 0); var bytesCount = _rowSize * elementsCount; sequence.Slice(0, bytesCount).CopyTo(_buffer.Span.Slice(_position)); _position += bytesCount; return new SequenceSize(bytesCount, elementsCount); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return new FixedStringTableColumn(_buffer, _rowSize, settings?.StringEncoding ?? Encoding.UTF8); } } private sealed class FixedStringBytesColumnWriter : FixedStringColumnWriterBase { private readonly IReadOnlyList> _rows; protected override int RowCount => _rows.Count; public FixedStringBytesColumnWriter(string columnName, string columnType, IReadOnlyList rows, int length) : this(columnName, columnType, MappedReadOnlyList>.Map(rows, b => b.AsMemory()), length) { } public FixedStringBytesColumnWriter(string columnName, string columnType, IReadOnlyList> rows, int length) : this(columnName, columnType, MappedReadOnlyList, ReadOnlyMemory>.Map(rows, m => m), length) { } public FixedStringBytesColumnWriter(string columnName, string columnType, IReadOnlyList> rows, int length) : base(columnName, columnType, length) { _rows = rows; } protected override int GetBytes(int position, Span buffer) { var bytes = _rows[position]; if (bytes.Length > buffer.Length) throw new InvalidCastException($"The length of the array ({bytes.Length}) is greater than the maximum length ({buffer.Length})."); bytes.Span.CopyTo(buffer); return bytes.Length; } } private sealed class FixedStringStringColumnWriter : FixedStringColumnWriterBase { private readonly IReadOnlyList> _rows; private readonly Encoding _stringEncoding; protected override int RowCount => _rows.Count; public FixedStringStringColumnWriter(string columnName, string columnType, IReadOnlyList> rows, int length, Encoding? stringEncoding) : this(columnName, columnType, MappedReadOnlyList, ReadOnlyMemory>.Map(rows, m => m), length, stringEncoding) { } public FixedStringStringColumnWriter(string columnName, string columnType, IReadOnlyList rows, int length, Encoding? stringEncoding) : this(columnName, columnType, MappedReadOnlyList>.Map(rows, str => str.AsMemory()), length, stringEncoding) { } public FixedStringStringColumnWriter(string columnName, string columnType, IReadOnlyList> rows, int length, Encoding? stringEncoding) : base(columnName, columnType, length) { _rows = rows; _stringEncoding = stringEncoding ?? Encoding.UTF8; } protected override int GetBytes(int position, Span buffer) { var str = _rows[position].Span; var bytesCount = _stringEncoding.GetByteCount(str); if (bytesCount == 0) return 0; if (bytesCount <= buffer.Length) { _stringEncoding.GetBytes(str, buffer); return bytesCount; } throw new InvalidCastException($"The length of the string ({bytesCount}) is greater than the maximum length ({buffer.Length})."); } } private abstract class FixedStringColumnWriterBase : IClickHouseColumnWriter { private readonly int _length; public string ColumnName { get; } public string ColumnType { get; } protected abstract int RowCount { get; } private int _position; public FixedStringColumnWriterBase(string columnName, string columnType, int length) { _length = length; ColumnName = columnName ?? throw new ArgumentNullException(nameof(columnName)); ColumnType = columnType; } public SequenceSize WriteNext(Span writeTo) { var size = Math.Min(RowCount - _position, writeTo.Length / _length); ReadOnlySpan zeroSpan = new byte[_length]; var span = writeTo; for (int i = 0; i < size; i++, span = span.Slice(_length)) { var bytesCount = GetBytes(_position++, span.Slice(0, _length)); zeroSpan.Slice(bytesCount).CopyTo(span.Slice(bytesCount)); } return new SequenceSize(size * _length, size); } protected abstract int GetBytes(int position, Span buffer); } private class FixedStringParameterWriter : IClickHouseParameterWriter> { private readonly FixedStringTypeInfo _type; public FixedStringParameterWriter(FixedStringTypeInfo type) { _type = type; } public bool TryCreateParameterValueWriter(ReadOnlyMemory value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var writer = new StringLiteralValueWriter(value, isNested); Debug.Assert(_type._length != null); if (writer.Length - 2 > _type._length.Value) ValidateLength(value); // Validate the length of the unescaped string without quota signs valueWriter = writer; return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlyMemory value) { ValidateLength(value); return StringParameterWriter.Interpolate(queryBuilder, value.Span); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _type, FunctionHelper.Apply); } private void ValidateLength(ReadOnlyMemory value) { Debug.Assert(_type._length != null); var length = _type._length.Value; var encoding = Encoding.UTF8; var bytesCount = encoding.GetByteCount(value.Span); if (bytesCount > length) throw new ClickHouseException(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"The length of the string ({bytesCount}) is greater than the maximum length ({length})."); } } private sealed class FixedStringParameterWriter : FixedStringParameterWriter, IClickHouseParameterWriter { private readonly Func> _convert; public FixedStringParameterWriter(FixedStringTypeInfo type, Func> convert) : base(type) { _convert = convert; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { return TryCreateParameterValueWriter(_convert(value), isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { return Interpolate(queryBuilder, _convert(value)); } } private class FixedStringHexParameterWriter : IClickHouseParameterWriter> { private readonly FixedStringTypeInfo _type; public FixedStringHexParameterWriter(FixedStringTypeInfo type) { _type = type; } public bool TryCreateParameterValueWriter(ReadOnlyMemory value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { ValidateLength(value); valueWriter = new HexStringLiteralValueWriter(value, isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlyMemory value) { ValidateLength(value); return HexStringParameterWriter.Interpolate(queryBuilder, value.Span); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _type, FunctionHelper.Apply); } private void ValidateLength(ReadOnlyMemory value) { Debug.Assert(_type._length != null); var length = _type._length.Value; if (value.Length > length) throw new ClickHouseException(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, $"The length of the array ({value.Length}) is greater than the maximum length ({length})."); } } private sealed class FixedStringHexParameterWriter : FixedStringHexParameterWriter, IClickHouseParameterWriter { private readonly Func> _convert; public FixedStringHexParameterWriter(FixedStringTypeInfo type, Func> convert) : base(type) { _convert = convert; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { return TryCreateParameterValueWriter(_convert(value), isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { return Interpolate(queryBuilder, _convert(value)); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Float32TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Float32TableColumn : StructureTableColumn { public Float32TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(double)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(double?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Float32TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal sealed class Float32TypeInfo : SimpleTypeInfo { public Float32TypeInfo() : base("Float32") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Float32Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(float), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) != typeof(float)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Float32Writer(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { float _ => HexStringParameterWriter.Create(this), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."), }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(float); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Single; } private sealed class Float32Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Float32Reader(int rowCount) : base(sizeof(float), rowCount) { } protected override float ReadElement(ReadOnlySpan source) { return BitConverter.ToSingle(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Float32TableColumn(buffer); } } private sealed class Float32Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public Float32Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(float), rows) { } protected override void WriteElement(Span writeTo, in float value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Float64TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class Float64TypeInfo : SimpleTypeInfo { public Float64TypeInfo() : base("Float64") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Float64Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(double), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { IReadOnlyList doubleRows; if (typeof(T) == typeof(double)) doubleRows = (IReadOnlyList)rows; else if (typeof(T) == typeof(float)) doubleRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Float64Writer(columnName, ComplexTypeName, doubleRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values"); object writer = default(T) switch { double _ => HexStringParameterWriter.Create(this), float _ => HexStringParameterWriter.Create(this, v => v), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."), }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(double); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Double; } private sealed class Float64Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Float64Reader(int rowCount) : base(sizeof(double), rowCount) { } protected override double ReadElement(ReadOnlySpan source) { return BitConverter.ToDouble(source); } } private sealed class Float64Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public Float64Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(double), rows) { } protected override void WriteElement(Span writeTo, in double value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/HexStringLiteralValueWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using System; using System.Diagnostics; namespace Octonica.ClickHouseClient.Types { internal sealed class HexStringLiteralValueWriter : IClickHouseParameterValueWriter { public const string HexDigits = "0123456789ABCDEF"; private readonly ReadOnlyMemory _value; private readonly bool _includeQuotes; public int Length { get; } public HexStringLiteralValueWriter(ReadOnlyMemory value, bool includeQuotes) { _value = value; _includeQuotes = includeQuotes; if (_includeQuotes) Length = 4 * value.Length + 4; else Length= 4 * _value.Length; } public int Write(Memory buffer) { Debug.Assert(buffer.Length >= Length); int count = 0; if (_includeQuotes) { buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\''; } foreach (var byteValue in _value.Span) { switch (byteValue) { case (byte)'\t': buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\t'; break; case (byte)'\n': buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\n'; break; case (byte)'\\': buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\\'; break; default: buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'x'; buffer.Span[count++] = (byte)HexDigits[byteValue >> 4]; buffer.Span[count++] = (byte)HexDigits[byteValue & 0xF]; break; } } if (_includeQuotes) { buffer.Span[count++] = (byte)'\\'; buffer.Span[count++] = (byte)'\''; } Debug.Assert(count == Length); return count; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/HexStringLiteralWriterCastMode.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal enum HexStringLiteralWriterCastMode { None = 0, Reinterpret = 1, Cast = 2 } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/HexStringParameterWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.InteropServices; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class HexStringParameterWriter : IClickHouseParameterWriter> { private const string HexDigits = HexStringLiteralValueWriter.HexDigits; private readonly IClickHouseColumnTypeInfo _typeInfo; public HexStringParameterWriter(IClickHouseColumnTypeInfo typeInfo) { _typeInfo = typeInfo; } public bool TryCreateParameterValueWriter(ReadOnlyMemory value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = new HexStringLiteralValueWriter(value, isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlyMemory value) { return Interpolate(queryBuilder, value.Span); } public static StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlySpan value) { queryBuilder.Append('\''); foreach (var byteValue in value) { queryBuilder.Append("\\x"); queryBuilder.Append(HexDigits[byteValue >> 4]); queryBuilder.Append(HexDigits[byteValue & 0xF]); } return queryBuilder.Append('\''); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); } public static HexStringParameterWriter Create(IClickHouseColumnTypeInfo typeInfo) where T : struct { var dummy = default(T); var dummyBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref dummy, 1)); var binaryTypeName = $"FixedString({dummyBytes.Length.ToString(CultureInfo.InvariantCulture)})"; return new HexStringParameterWriter(typeInfo, HexStringLiteralWriterCastMode.Reinterpret, binaryTypeName, Convert); } public static HexStringParameterWriter Create(IClickHouseColumnTypeInfo typeInfo, Func convert) where TOut : struct { var dummy = default(TOut); var dummyBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref dummy, 1)); var binaryTypeName = $"FixedString({dummyBytes.Length.ToString(CultureInfo.InvariantCulture)})"; return new HexStringParameterWriter(typeInfo, HexStringLiteralWriterCastMode.Reinterpret, binaryTypeName, v => Convert(convert(v))); } private static ReadOnlyMemory Convert(T value) where T : struct { var src = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref value, 1)); var dst = new byte[src.Length]; src.CopyTo(dst); return dst; } } internal sealed class HexStringParameterWriter : IClickHouseParameterWriter { private readonly IClickHouseColumnTypeInfo _typeInfo; private readonly HexStringLiteralWriterCastMode _castMode; private readonly string? _valueType; private readonly Func> _convert; public HexStringParameterWriter(IClickHouseColumnTypeInfo typeInfo, Func> convert) : this(typeInfo, HexStringLiteralWriterCastMode.None, null, convert) { } public HexStringParameterWriter(IClickHouseColumnTypeInfo typeInfo, HexStringLiteralWriterCastMode castMode, string? valueType, Func> convert) { _typeInfo = typeInfo; _castMode = castMode; _valueType = valueType; _convert = convert; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var bytes = _convert(value); valueWriter = new HexStringLiteralValueWriter(bytes, isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { var bytes = _convert(value); switch (_castMode) { case HexStringLiteralWriterCastMode.Cast: queryBuilder.Append("CAST("); break; case HexStringLiteralWriterCastMode.Reinterpret: queryBuilder.Append("reinterpret("); break; case HexStringLiteralWriterCastMode.None: break; default: throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Unknown value cast mode: \"{_castMode}\"."); } if (_valueType != null) queryBuilder.Append("CAST("); HexStringParameterWriter.Interpolate(queryBuilder, bytes.Span); if (_valueType != null) queryBuilder.Append(" AS ").Append(_valueType).Append(')'); switch (_castMode) { case HexStringLiteralWriterCastMode.Cast: queryBuilder.Append(" AS ").Append(_typeInfo.ComplexTypeName).Append(')'); break; case HexStringLiteralWriterCastMode.Reinterpret: queryBuilder.Append(",'").Append(_typeInfo.ComplexTypeName.Replace("'", "''")).Append("')"); break; } return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { if (_valueType == null) return writeValue(queryBuilder, _typeInfo, FunctionHelper.Apply); var valueType = typeInfoProvider.GetTypeInfo(_valueType); var castMode = _castMode; if (_typeInfo.ComplexTypeName == valueType.ComplexTypeName) castMode = HexStringLiteralWriterCastMode.None; switch (castMode) { case HexStringLiteralWriterCastMode.Cast: return writeValue(queryBuilder, valueType, (qb, realWrite) => { qb.Append('('); realWrite(qb); return qb.Append("::").Append(_typeInfo.ComplexTypeName).Append(')'); }); case HexStringLiteralWriterCastMode.Reinterpret: return writeValue(queryBuilder, valueType, (qb, realWrite) => { qb.Append("reinterpret("); realWrite(qb); return qb.Append(",'").Append(_typeInfo.ComplexTypeName.Replace("'", "''")).Append("')"); }); default: Debug.Assert(castMode == HexStringLiteralWriterCastMode.None); return writeValue(queryBuilder, valueType, FunctionHelper.Apply); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseColumnReinterpreter.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal interface IClickHouseColumnReinterpreter { Type? BuiltInConvertToType { get; } Type? ExternalConvertToType { get; } IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseColumnTypeDescriptor.cs ================================================ #region License Apache 2.0 /* Copyright 2021-2022 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { /// /// Represents a set of properties describing a type of a column. /// public interface IClickHouseColumnTypeDescriptor { /// /// Gets the type of the column. /// ClickHouseDbType? ClickHouseDbType { get; } /// /// Gets the type of the column's value (i.e. the type of column's cells). /// Type ValueType { get; } /// /// Gets the value indicating whether the column can contain NULLs. /// bool? IsNullable { get; } /// /// Gets the size. This value is applied to the ClickHouse type FixedString. /// int Size { get; } /// /// Gets the precision. This value is applied to ClickHouse types Decimal and DateTime64. /// byte? Precision { get; } /// /// Gets the scale. This value is applied to the ClickHouse type Decimal. /// byte? Scale { get; } /// /// Gets the time zone. This value is applied to ClickHouse types DateTime and DateTime64. /// TimeZoneInfo? TimeZone { get; } /// /// Gets the rank (a number of dimensions) of an array. /// int? ArrayRank { get; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseColumnTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { /// /// Represents basic information about the ClickHouse type. Provides access to factory methods for creating column readers and writers. /// /// /// Being a part of the ClickHouseClient's infrastructure, the interface is considered unstable. It can be changed between minor versions. /// public interface IClickHouseColumnTypeInfo : IClickHouseTypeInfo { /// /// Creates and returns a new instance of configured to read the specified number of rows. /// /// The number of rows that the reader should read. /// The that should read the specified number of rows. IClickHouseColumnReader CreateColumnReader(int rowCount); /// /// Creates and returns a new instance of configured to read the specified number of rows. /// /// The number of rows that the reader should read. /// /// One of supported serialization modes (see for details). /// When the mode is an implementation /// of this method must call . /// /// The that should read the specified number of rows. IClickHouseColumnReader CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { switch (serializationMode) { case ClickHouseColumnSerializationMode.Default: return CreateColumnReader(rowCount); case ClickHouseColumnSerializationMode.Sparse: case ClickHouseColumnSerializationMode.Custom: return new CustomSerializationColumnReader(this, rowCount, serializationMode); default: throw new ArgumentException($"Unknown serialization mode: {serializationMode}.", nameof(serializationMode)); } } /// /// Creates and returns a new instance of configured to skip the specified number of rows. /// /// The number of rows that the reader should skip. /// The that should skip the specified number of rows. IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount); /// /// Creates and returns a new instance of configured to skip the specified number of rows. /// /// The number of rows that the reader should skip. /// /// One of supported serialization modes (see for details). /// When the mode is an implementation /// of this method must call . /// /// The that should skip the specified number of rows. IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { switch (serializationMode) { case ClickHouseColumnSerializationMode.Default: return CreateSkippingColumnReader(rowCount); case ClickHouseColumnSerializationMode.Sparse: case ClickHouseColumnSerializationMode.Custom: return new CustomSerializationSkippingColumnReader(this, rowCount, serializationMode); default: throw new ArgumentException($"Unknown serialization mode: {serializationMode}.", nameof(serializationMode)); } } /// /// Creates and returns a new instance of that can write specified rows to a binary stream. /// /// The type of the list of rows. /// The name of the column. /// The list of rows. /// Optional argument. Additional settings for the column writer. /// The that can write specified rows to a binary stream IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings); /// /// Returns an instance of based on this type but with the specified list of type arguments. /// /// The list of strings. Each string in the list describes an argument of the type. /// The type provider that can be used to get other types. /// The with the same as this type and with the specified list of type arguments IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider); /// /// Creates an instance of capable of writing a value of the type /// as a ClickHouse parameter. /// /// The that can writer the value of the type as a parameter. IClickHouseParameterWriter CreateParameterWriter(); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseConfigurableTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { /// /// Represents basic information about the ClickHouse type that depends on the server's settings. /// Provides access to factory methods for creating column readers and writers. /// public interface IClickHouseConfigurableTypeInfo : IClickHouseColumnTypeInfo { /// /// Returns the that represents the type with the specified settings. /// /// Information about the server. /// The that represents the type with the specified settings. IClickHouseColumnTypeInfo Configure(ClickHouseServerInfo serverInfo); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseEnumConverter.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { /// /// The basic interface for enum converters. Objects that implement this interface are responsible for the enum's type dispatching. /// public interface IClickHouseEnumConverter { /// /// When implemented in a derrived class must call /// with an appropriate generic argument. /// /// The type of the returned value. /// The dispatcher that requires the type of enum to be passed as the generic parameter. /// The value returned by the dispatcher. public T Dispatch(IClickHouseEnumConverterDispatcher dispatcher); } /// /// The basic interface for enum converters that can convert from and to .NET enums. /// Objects that implement this interface are responsible for the enum's type dispatching. /// /// The type of the enum. public interface IClickHouseEnumConverter : IClickHouseEnumConverter where TEnum : Enum { /// /// When implemented in a derived class returns a enum's value corresponding to a numeric value of the ClickHouse enum, a string value of the ClickHouse enum, or both. /// /// The numeric value of the ClickHouse enum. /// The string value of the ClickHouse enum. /// When this method returns, contains the value of enum or the default value of when returns . /// if there are 's value corresponding to the specified ClickHouse enum; otherwise . bool TryMap(int value, string stringValue, [NotNullWhen(true)] out TEnum enumValue); } /// /// The interface for a enum dispatchers. /// /// The type of an object returned by this despatcher. public interface IClickHouseEnumConverterDispatcher { /// /// When implemented in a derrived class executes an arbitrary operation with respect to the enum converter /// and returns the result of this operation. /// /// The type of the enum. /// The enum converter for the enum of the specified type. /// The result of an executed operations. /// This method can be viewed as a closure of the type argument. public T Dispatch(IClickHouseEnumConverter enumConverter) where TEnum : Enum; } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseReinterpretedTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal interface IClickHouseReinterpretedTableColumn: IClickHouseTableColumn { IClickHouseReinterpretedTableColumn Chain(Func reinterpret); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { /// /// Represents basic information about the ClickHouse type. /// public interface IClickHouseTypeInfo { /// /// Gets the full name of the type. The full name contains of the name of the type and the list of arguments. /// string ComplexTypeName { get; } /// /// Gets the name of the type without a list of arguments. /// string TypeName { get; } /// /// Gets the number of generic arguments in the list of arguments. Only arguments that are types themselves are counted as generics. /// int GenericArgumentsCount { get; } /// /// Gets the number of arguments in the list of arguments. /// int TypeArgumentsCount => GenericArgumentsCount; /// /// Gets the CLR type to which this ClickHouse type is mapped. /// /// The to which this ClickHouse type is mapped. /// If the ClickHouse type is mapped to a nullable structure this method returns (). Type GetFieldType(); /// /// Gets the type code specific to ClickHouse. /// ClickHouseDbType GetDbType(); /// /// Gets the generic arguments at the specified position. Generic arguments are counted separately, /// which means that the index can't be greater than . /// /// The zero-based index of the generic argument. It can't be greater than . /// The which represents the generic argument. IClickHouseTypeInfo GetGenericArgument(int index); /// /// Gets the argument of the type. For generic arguments this method returns an . /// /// The zero-based index of the argument. It can't be greater than . /// The argument of the type. object GetTypeArgument(int index) { if (TypeArgumentsCount == 0) throw new NotSupportedException($"The type \"{TypeName}\" doesn't have arguments."); return GetGenericArgument(index); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IClickHouseTypeInfoProvider.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2022 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { /// /// The base interface for an object that provides information about supported types. /// public interface IClickHouseTypeInfoProvider { /// /// Gets the type by its name. /// /// The name of the type. /// The that provides information about the type. IClickHouseColumnTypeInfo GetTypeInfo(string typeName); /// IClickHouseColumnTypeInfo GetTypeInfo(ReadOnlyMemory typeName); /// /// Gets the type based on the . /// /// The descriptor of a column's type. /// The that provides information about the type. IClickHouseColumnTypeInfo GetTypeInfo(IClickHouseColumnTypeDescriptor typeDescriptor); /// /// Returns the that provides access to types configured with the specified settings. /// /// Information about the server. /// The that provides access to types configured with the specified settings. IClickHouseTypeInfoProvider Configure(ClickHouseServerInfo serverInfo); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int128TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal sealed class Int128TypeInfo : BigIntegerTypeInfoBase { public Int128TypeInfo() : base("Int128", 128 / 8, false) { } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Int128; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int16TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Int16TableColumn : StructureTableColumn { public Int16TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(int)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(int?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int16TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class Int16TypeInfo : SimpleTypeInfo { public Int16TypeInfo() : base("Int16") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Int16Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(short), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList shortRows; if (type == typeof(short)) shortRows = (IReadOnlyList)rows; else if (type == typeof(sbyte)) shortRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) shortRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Int16Writer(columnName, ComplexTypeName, shortRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { short _ => new SimpleParameterWriter(this, appendTypeCast: true), sbyte _ => new SimpleParameterWriter(this, appendTypeCast: true), byte _ => new SimpleParameterWriter(this, appendTypeCast: true), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(short); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Int16; } internal sealed class Int16Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Int16Reader(int rowCount) : base(sizeof(short), rowCount) { } protected override short ReadElement(ReadOnlySpan source) { return BitConverter.ToInt16(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Int16TableColumn(buffer); } } internal sealed class Int16Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public Int16Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(short), rows) { } protected override void WriteElement(Span writeTo, in short value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int256TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal sealed class Int256TypeInfo : BigIntegerTypeInfoBase { public Int256TypeInfo() : base("Int256", 256 / 8, false) { } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Int256; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int32TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Int32TableColumn : StructureTableColumn { public Int32TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int32TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class Int32TypeInfo : SimpleTypeInfo { public Int32TypeInfo() : base("Int32") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Int32Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(int), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList intRows; if (type == typeof(int)) intRows = (IReadOnlyList)rows; else if (type == typeof(short)) intRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(ushort)) intRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(sbyte)) intRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) intRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Int32Writer(columnName, ComplexTypeName, intRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { int _ => new SimpleParameterWriter(this, appendTypeCast: true), short _ => new SimpleParameterWriter(this, appendTypeCast: true), ushort _ => new SimpleParameterWriter(this, appendTypeCast: true), sbyte _ => new SimpleParameterWriter(this, appendTypeCast: true), byte _ => new SimpleParameterWriter(this, appendTypeCast: true), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(int); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Int32; } private sealed class Int32Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Int32Reader(int rowCount) : base(sizeof(int), rowCount) { } protected override int ReadElement(ReadOnlySpan source) { return BitConverter.ToInt32(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Int32TableColumn(buffer); } } private sealed class Int32Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public Int32Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(int), rows) { } protected override void WriteElement(Span writeTo, in int value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int64TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class Int64TypeInfo : SimpleTypeInfo { public Int64TypeInfo() : base("Int64") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Int64Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(long), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList longRows; if (type == typeof(long)) longRows = (IReadOnlyList)rows; else if (type == typeof(int)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(uint)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(short)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(ushort)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(sbyte)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) longRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Int64Writer(columnName, ComplexTypeName, longRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { long _ => new SimpleParameterWriter(this, appendTypeCast: true), int _ => new SimpleParameterWriter(this, appendTypeCast: true), short _ => new SimpleParameterWriter(this, appendTypeCast: true), ushort _ => new SimpleParameterWriter(this, appendTypeCast: true), sbyte _ => new SimpleParameterWriter(this, appendTypeCast: true), byte _ => new SimpleParameterWriter(this, appendTypeCast: true), _ => throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."), }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(long); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Int64; } private sealed class Int64Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Int64Reader(int rowCount) : base(sizeof(long), rowCount) { } protected override long ReadElement(ReadOnlySpan source) { return BitConverter.ToInt64(source); } } private sealed class Int64Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public Int64Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(long), rows) { } protected override void WriteElement(Span writeTo, in long value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int8TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class Int8TableColumn : StructureTableColumn { public Int8TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(short)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(int)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(short?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(int?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/Int8TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal sealed class Int8TypeInfo : SimpleTypeInfo { public Int8TypeInfo() : base("Int8") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new Int8Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(sbyte), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) != typeof(sbyte)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new Int8Writer(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); if (type == typeof(sbyte)) return (IClickHouseParameterWriter)(object)new SimpleParameterWriter(this, appendTypeCast: true); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public override Type GetFieldType() { return typeof(sbyte); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.SByte; } internal sealed class Int8Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public Int8Reader(int rowCount) : base(sizeof(sbyte), rowCount) { } protected override sbyte ReadElement(ReadOnlySpan source) { return unchecked((sbyte) source[0]); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new Int8TableColumn(buffer); } } internal sealed class Int8Writer : IClickHouseColumnWriter { private readonly IReadOnlyList _rows; private int _position; public string ColumnName { get; } public string ColumnType { get; } public Int8Writer(string columnName, string columnType, IReadOnlyList rows) { _rows = rows; ColumnName = columnName; ColumnType = columnType; } public SequenceSize WriteNext(Span writeTo) { var size = Math.Min(writeTo.Length, _rows.Count - _position); for (int i = 0; i < size; i++, _position++) writeTo[i] = unchecked((byte) _rows[_position]); return new SequenceSize(size, size); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IntermediateClickHouseTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal readonly struct IntermediateClickHouseTypeInfo { public ClickHouseDbType DbType { get; } public string ClickHouseType { get; } public bool IsNullable { get; } public int ArrayRank { get; } public IntermediateClickHouseTypeInfo(ClickHouseDbType dbType, string clickHouseType, bool isNullable, int arrayRank) { DbType = dbType; ClickHouseType = clickHouseType; IsNullable = isNullable; ArrayRank = arrayRank; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IpColumnReaderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Net; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal abstract class IpColumnReaderBase : IClickHouseColumnReader { private readonly int _rowCount; private readonly Memory _buffer; private int _position; public int ElementSize { get; } protected IpColumnReaderBase(int rowCount, int elementSize) { _rowCount = rowCount; ElementSize = elementSize; _buffer = new Memory(new byte[_rowCount * elementSize]); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var byteSize = Math.Min(ElementSize * (_rowCount - _position), (int) (sequence.Length - sequence.Length % ElementSize)); var elementCount = byteSize / ElementSize; if (elementCount == 0) return new SequenceSize(0, 0); sequence.Slice(0, byteSize).CopyTo(_buffer.Slice(_position * ElementSize).Span); _position += elementCount; return new SequenceSize(byteSize, elementCount); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return EndRead(_buffer.Slice(0, _position * ElementSize)); } protected abstract IClickHouseTableColumn EndRead(ReadOnlyMemory buffer); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IpV4TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Net; using System.Runtime.InteropServices; namespace Octonica.ClickHouseClient.Types { internal sealed class IpV4TableColumn: IClickHouseTableColumn { private readonly ReadOnlyMemory _buffer; public int RowCount => _buffer.Length / sizeof(uint); public IPAddress DefaultValue => IPAddress.Any; public IpV4TableColumn(ReadOnlyMemory buffer) { _buffer = buffer; } public bool IsNull(int index) { return false; } public IPAddress GetValue(int index) { Span address = stackalloc int[1]; var addressSpan = MemoryMarshal.AsBytes(address); _buffer.Slice(index * sizeof(uint), sizeof(uint)).Span.CopyTo(addressSpan); address[0] = IPAddress.NetworkToHostOrder(address[0]); return new IPAddress(addressSpan); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { IClickHouseTableColumn? result = null; if (typeof(T) == typeof(IPAddress)) { result = this; } else if (typeof(T) == typeof(string)) { result = new ReinterpretedTableColumn(this, v => v.ToString()); } if (typeof(T) == typeof(uint)) { result = new ReinterpretedTableColumn(this, new RawIpV4TableColumn(_buffer, m => BitConverter.ToUInt32(m.Span))); } else if (typeof(T) == typeof(uint?)) { result = new ReinterpretedTableColumn(this, new NullableStructTableColumn(null, new RawIpV4TableColumn(_buffer, m => BitConverter.ToUInt32(m.Span)))); } else if (typeof(T) == typeof(int)) { result = new ReinterpretedTableColumn(this, new RawIpV4TableColumn(_buffer, m => BitConverter.ToInt32(m.Span))); } else if (typeof(T) == typeof(int?)) { result = new ReinterpretedTableColumn(this, new NullableStructTableColumn(null, new RawIpV4TableColumn(_buffer, m => BitConverter.ToInt32(m.Span)))); } return (IClickHouseTableColumn?) result; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } private sealed class RawIpV4TableColumn : IClickHouseTableColumn where TStruct : struct { private readonly ReadOnlyMemory _buffer; private readonly Func, TStruct> _getValue; public int RowCount => _buffer.Length / sizeof(uint); public TStruct DefaultValue => default; public RawIpV4TableColumn(ReadOnlyMemory buffer, Func, TStruct> getValue) { _buffer = buffer; _getValue = getValue; } public bool IsNull(int index) { return false; } public TStruct GetValue(int index) { return _getValue(_buffer.Slice(index * sizeof(uint), sizeof(uint))); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return this as IClickHouseTableColumn; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IpV4TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Runtime.InteropServices; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class IpV4TypeInfo : SimpleTypeInfo { public IpV4TypeInfo() : base("IPv4") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new IpV4Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(uint), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList preparedRows; if (typeof(IPAddress).IsAssignableFrom(type)) preparedRows = MappedReadOnlyList.Map((IReadOnlyList)rows, IpAddressToUInt32); else if (type == typeof(string)) preparedRows = MappedReadOnlyList.Map((IReadOnlyList)rows, IpAddressStringToUInt32); else if (type == typeof(uint)) preparedRows = (IReadOnlyList)rows; else if (type == typeof(int)) preparedRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => unchecked((uint)v)); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new IpV4Writer(columnName, TypeName, preparedRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); const string valueType = "UInt32"; object writer; if (type == typeof(IPAddress)) writer = new SimpleParameterWriter(valueType, this, null, true, IpAddressToUInt32); else if (type == typeof(string)) writer = new SimpleParameterWriter(valueType, this, null, true, IpAddressStringToUInt32); else if (type == typeof(uint)) writer = new SimpleParameterWriter(valueType, this, appendTypeCast: true); else if (type == typeof(int)) writer = new SimpleParameterWriter(valueType, this, null, true, v => unchecked((uint)v)); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(IPAddress); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.IpV4; } private static uint IpAddressStringToUInt32(string? address) { if (address == null) return 0; if (!IPAddress.TryParse(address, out var ipAddress)) throw new InvalidCastException($"The string \"{address}\" is not a valid IPv4 address."); return IpAddressToUInt32(ipAddress); } private static uint IpAddressToUInt32(IPAddress? address) { if (address == null) return 0; if (address.AddressFamily == AddressFamily.InterNetworkV6 && address.IsIPv4MappedToIPv6) address = address.MapToIPv4(); if (address.AddressFamily != AddressFamily.InterNetwork) throw new InvalidCastException($"The network address \"{address}\" is not a IPv4 address."); Span result = stackalloc uint[1]; if (!address.TryWriteBytes(MemoryMarshal.AsBytes(result), out var written) || written != sizeof(uint)) throw new InvalidCastException($"The network address \"{address}\" is not a IPv4 address."); return unchecked((uint) IPAddress.HostToNetworkOrder((int) result[0])); } private sealed class IpV4Reader : IpColumnReaderBase { public IpV4Reader(int rowCount) : base(rowCount, sizeof(uint)) { } protected override IClickHouseTableColumn EndRead(ReadOnlyMemory buffer) { return new IpV4TableColumn(buffer); } } private sealed class IpV4Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public IpV4Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(uint), rows) { } protected override void WriteElement(Span writeTo, in uint value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IpV6TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Net; namespace Octonica.ClickHouseClient.Types { internal sealed class IpV6TableColumn : IClickHouseTableColumn { private readonly ReadOnlyMemory _buffer; public int RowCount => _buffer.Length / 16; public IPAddress DefaultValue => IPAddress.IPv6Any; public IpV6TableColumn(ReadOnlyMemory buffer) { _buffer = buffer; } public bool IsNull(int index) { return false; } public IPAddress GetValue(int index) { return new IPAddress(_buffer.Span.Slice(index * 16, 16)); } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(string)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v.ToString()); return this as IClickHouseTableColumn; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/IpV6TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Net; using System.Net.Sockets; using System.Runtime.CompilerServices; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class IpV6TypeInfo: SimpleTypeInfo { private const int AddressSize = 16; public IpV6TypeInfo() : base("IPv6") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new IpV6Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(AddressSize, rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList preparedRows; if(typeof(IPAddress).IsAssignableFrom(type)) preparedRows = (IReadOnlyList)rows; else if(type == typeof(string)) preparedRows = MappedReadOnlyList.Map((IReadOnlyList)rows, ParseIpAddress); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new IpV6Writer(columnName, TypeName, preparedRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (typeof(T) == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); var binaryTypeName = $"FixedString({AddressSize.ToString(CultureInfo.InvariantCulture)})"; object writer; if (type == typeof(IPAddress)) writer = new HexStringParameterWriter(this, HexStringLiteralWriterCastMode.Cast, binaryTypeName, theValue => GetBytes(theValue)); else if (type == typeof(string)) writer = new HexStringParameterWriter(this, HexStringLiteralWriterCastMode.Cast, binaryTypeName, theValue => GetBytes(ParseIpAddress(theValue))); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(IPAddress); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.IpV6; } [return: NotNullIfNotNull("address")] private static IPAddress? ParseIpAddress(string? address) { if (address == null) return null; if (!IPAddress.TryParse(address, out var ipAddress)) throw new InvalidCastException($"The string \"{address}\" is not a valid IPv4 address."); return ipAddress; } private static byte[] GetBytes(IPAddress ipAddress) { var buffer = new byte[AddressSize]; WriteBytes(buffer, ipAddress); return buffer; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void WriteBytes(Span writeTo, IPAddress ipAddress) { if (ipAddress.AddressFamily == AddressFamily.InterNetwork) ipAddress = ipAddress.MapToIPv6(); if (ipAddress.AddressFamily != AddressFamily.InterNetworkV6) throw new InvalidCastException($"The network address \"{ipAddress}\" is not a IPv6 address."); if (!ipAddress.TryWriteBytes(writeTo, out var bytesWritten) || bytesWritten != AddressSize) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error: IPv6 address writing error."); } private sealed class IpV6Reader : IpColumnReaderBase { public IpV6Reader(int rowCount) : base(rowCount, AddressSize) { } protected override IClickHouseTableColumn EndRead(ReadOnlyMemory buffer) { return new IpV6TableColumn(buffer); } } private sealed class IpV6Writer : IClickHouseColumnWriter { private readonly IReadOnlyList _rows; private int _position; public string ColumnName { get; } public string ColumnType { get; } public IpV6Writer(string columnName, string columnType, IReadOnlyList rows) { _rows = rows; ColumnName = columnName; ColumnType = columnType; } public SequenceSize WriteNext(Span writeTo) { var elementsCount = Math.Min(_rows.Count - _position, writeTo.Length / AddressSize); for (int i = 0; i < elementsCount; i++, _position++) { var ipAddress = _rows[_position]; if (ipAddress == null) { writeTo.Slice(i * AddressSize, AddressSize).Fill(0); continue; } WriteBytes(writeTo.Slice(i * AddressSize), ipAddress); } return new SequenceSize(elementsCount * AddressSize, elementsCount); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/KeyValuePairTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; using System.Diagnostics; namespace Octonica.ClickHouseClient.Types { internal sealed class KeyValuePairTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _keyColumn; private readonly IClickHouseTableColumn _valueColumn; public KeyValuePair DefaultValue => new KeyValuePair(_keyColumn.DefaultValue, _valueColumn.DefaultValue); public KeyValuePairTableColumn(int rowCount, IClickHouseTableColumn keyColumn, IClickHouseTableColumn valueColumn) : base(rowCount) { _keyColumn = keyColumn; _valueColumn = valueColumn; } protected override object GetTupleValue(int index) { return GetValue(index); } public new KeyValuePair GetValue(int index) { CheckIndex(index); return new KeyValuePair(_keyColumn.GetValue(index), _valueColumn.GetValue(index)); } public override IEnumerable GetColumns() { yield return _keyColumn; yield return _valueColumn; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 2); var keyColumn = TryReinterpret(columns[0]); if (keyColumn == null) return null; var valueColumn = TryReinterpret(columns[1]); if (valueColumn == null) return null; return new KeyValuePairTableColumn(rowCount, keyColumn, valueColumn); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/LowCardinalityTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Types { internal sealed class LowCardinalityTableColumn : IClickHouseTableColumn { private readonly ReadOnlyMemory _keys; private readonly int _keySize; private readonly IClickHouseTableColumn _values; private readonly bool _isNullable; public int RowCount { get; } public LowCardinalityTableColumn(ReadOnlyMemory keys, int keySize, IClickHouseTableColumn values, bool isNullable) { _keys = keys; _keySize = keySize; _values = values; _isNullable = isNullable; RowCount = _keys.Length / _keySize; } public bool IsNull(int index) { if (!_isNullable) return false; var valueIndex = GetValueIndex(index); return valueIndex == 0; } public object GetValue(int index) { var valueIndex = GetValueIndex(index); if (valueIndex == 0 && _isNullable) return DBNull.Value; return _values.GetValue(valueIndex); } public IClickHouseTableColumn? TryReinterpret() { var reinterpretedValues = _values as IClickHouseTableColumn ?? _values.TryReinterpret(); if (reinterpretedValues == null) return null; return new LowCardinalityTableColumn(_keys, _keySize, reinterpretedValues, _isNullable); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { var reinterpretedValues = _values as IClickHouseArrayTableColumn ?? _values.TryReinterpretAsArray(); if (reinterpretedValues == null) return null; return new LowCardinalityArrayTableColumn(this, _keys, _keySize, reinterpretedValues, _isNullable); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } private int GetValueIndex(int index) { if (index < 0 || index > RowCount) throw new ArgumentOutOfRangeException(nameof(index)); int valueIndex = 0; var valueIndexBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref valueIndex, 1)).Slice(0, _keySize); _keys.Slice(index * _keySize, _keySize).Span.CopyTo(valueIndexBytes); return valueIndex; } } internal sealed class LowCardinalityTableColumn : IClickHouseTableColumn { private readonly ReadOnlyMemory _keys; private readonly int _keySize; private readonly IClickHouseTableColumn _values; private readonly bool _isNullable; public int RowCount { get; } public TValue DefaultValue => _values.DefaultValue; public LowCardinalityTableColumn(ReadOnlyMemory keys, int keySize, IClickHouseTableColumn values, bool isNullable) { _keys = keys; _keySize = keySize; _values = values; _isNullable = isNullable; RowCount = _keys.Length / _keySize; } public bool IsNull(int index) { if (!_isNullable) return false; var valueIndex = GetValueIndex(index); return valueIndex == 0; } public TValue GetValue(int index) { var valueIndex = GetValueIndex(index); if (valueIndex == 0 && _isNullable) { var defaultValue = default(TValue); if (!(defaultValue is null)) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Can't convert NULL to \"{typeof(TValue)}\"."); return defaultValue!; } return _values.GetValue(valueIndex); } object IClickHouseTableColumn.GetValue(int index) { var valueIndex = GetValueIndex(index); if (valueIndex == 0 && _isNullable) return DBNull.Value; return ((IClickHouseTableColumn) _values).GetValue(valueIndex); } public IClickHouseTableColumn? TryReinterpret() { var reinterpretedValues = _values as IClickHouseTableColumn ?? _values.TryReinterpret(); if (reinterpretedValues == null) return null; return new LowCardinalityTableColumn(_keys, _keySize, reinterpretedValues, _isNullable); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { var reinterpretedValues = _values as IClickHouseArrayTableColumn ?? _values.TryReinterpretAsArray(); if (reinterpretedValues == null) return null; return new LowCardinalityArrayTableColumn(this, _keys, _keySize, reinterpretedValues, _isNullable); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } private int GetValueIndex(int index) { if (index < 0 || index > RowCount) throw new ArgumentOutOfRangeException(nameof(index)); int valueIndex = 0; var valueIndexBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref valueIndex, 1)).Slice(0, _keySize); _keys.Slice(index * _keySize, _keySize).Span.CopyTo(valueIndexBytes); return valueIndex; } } internal sealed class LowCardinalityArrayTableColumn : IClickHouseArrayTableColumn { private readonly IClickHouseTableColumn _reinterpretationRoot; private readonly ReadOnlyMemory _keys; private readonly int _keySize; private readonly IClickHouseArrayTableColumn _values; private readonly bool _isNullable; public int RowCount { get; } public LowCardinalityArrayTableColumn(IClickHouseTableColumn reinterpretationRoot, ReadOnlyMemory keys, int keySize, IClickHouseArrayTableColumn values, bool isNullable) { _reinterpretationRoot = reinterpretationRoot; _keys = keys; _keySize = keySize; _values = values; _isNullable = isNullable; RowCount = _keys.Length / _keySize; } public int CopyTo(int index, Span buffer, int dataOffset) { var valueIndex = GetValueIndex(index); if (valueIndex == 0 && _isNullable) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Can't copy NULL value to the buffer."); return _values.CopyTo(valueIndex, buffer, dataOffset); } public object GetValue(int index) { var valueIndex = GetValueIndex(index); if (valueIndex == 0 && _isNullable) return DBNull.Value; return _values.GetValue(valueIndex); } public bool IsNull(int index) { if (!_isNullable) return false; var valueIndex = GetValueIndex(index); return valueIndex == 0; } public IClickHouseTableColumn? TryReinterpret() { return _reinterpretationRoot as IClickHouseTableColumn ?? _reinterpretationRoot.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return _reinterpretationRoot as IClickHouseArrayTableColumn ?? _reinterpretationRoot.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } private int GetValueIndex(int index) { if (index < 0 || index > RowCount) throw new ArgumentOutOfRangeException(nameof(index)); int valueIndex = 0; var valueIndexBytes = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref valueIndex, 1)).Slice(0, _keySize); _keys.Slice(index * _keySize, _keySize).Span.CopyTo(valueIndexBytes); return valueIndex; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/LowCardinalityTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class LowCardinalityTypeInfo : IClickHouseColumnTypeInfo { private readonly IClickHouseColumnTypeInfo? _baseType; public string ComplexTypeName { get; } public string TypeName => "LowCardinality"; public int GenericArgumentsCount => _baseType == null ? 0 : 1; public LowCardinalityTypeInfo() { _baseType = null; ComplexTypeName = TypeName; } private LowCardinalityTypeInfo(IClickHouseColumnTypeInfo baseType) { _baseType = baseType ?? throw new ArgumentNullException(nameof(baseType)); ComplexTypeName = $"{TypeName}({_baseType.ComplexTypeName})"; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { var typeInfo = GetBaseTypeInfo(); return new LowCardinalityColumnReader(rowCount, typeInfo.baseType, typeInfo.isNullable); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { var typeInfo = GetBaseTypeInfo(); return new LowCardinalitySkippingColumnReader(rowCount, typeInfo.baseType); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateSkippingColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } private (IClickHouseColumnTypeInfo baseType, bool isNullable) GetBaseTypeInfo() { if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); if (_baseType is NullableTypeInfo nullableBaseType) { if (nullableBaseType.UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{_baseType.ComplexTypeName}\" is not fully specified."); // LowCardinality column stores NULL as the key 0 return (nullableBaseType.UnderlyingType, true); } return (_baseType, false); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) == typeof(string)) { // In most cases values of type T can be inserted into a column of type LowCardinality(T). // However, a column of type Map(LowCardinality(String), TValue) doesn't accept keys of type String. // https://github.com/Octonica/ClickHouseClient/issues/86 return CreateColumnWriter(columnName, (IReadOnlyList)rows, StringComparer.Ordinal, string.Empty, columnSettings); } if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); // Writing values as is. Let the database do de-duplication return _baseType.CreateColumnWriter(columnName, rows, columnSettings); } private LowCardinalityColumnWriter CreateColumnWriter( string columnName, IReadOnlyList rows, IEqualityComparer equalityComparer, T defaultValue, ClickHouseColumnSettings? columnSettings) where T : notnull { var (baseType, isNullable) = GetBaseTypeInfo(); var map = new Dictionary(equalityComparer); var indices = new List(rows.Count); var uniqueValues = new List(); if (isNullable) uniqueValues.Add(defaultValue); // Reserve the first non-nullable position in the dictionary for the defualt value map.Add(defaultValue, uniqueValues.Count); uniqueValues.Add(defaultValue); foreach (var row in rows) { if (isNullable && row is null) { indices.Add(0); continue; } if (!map.TryGetValue(row, out var index)) { index = uniqueValues.Count; map.Add(row, index); uniqueValues.Add(row); } indices.Add(index); } if (indices.Count == 0) uniqueValues.Clear(); KeySizeCode keySizeCode; if (uniqueValues.Count > ushort.MaxValue) keySizeCode = KeySizeCode.UInt32; else if (uniqueValues.Count > byte.MaxValue) keySizeCode = KeySizeCode.UInt16; else keySizeCode = KeySizeCode.UInt8; Debug.Assert(_baseType != null); var baseWriter = baseType.CreateColumnWriter(columnName, uniqueValues, columnSettings); return new LowCardinalityColumnWriter(baseWriter, uniqueValues.Count, ComplexTypeName, indices, keySizeCode); } public IClickHouseParameterWriter CreateParameterWriter() { if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _baseType.CreateParameterWriter(); } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (_baseType != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); if (options.Count > 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); var baseType = typeInfoProvider.GetTypeInfo(options[0]); return new LowCardinalityTypeInfo(baseType); } public Type GetFieldType() { if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _baseType.GetFieldType(); } public ClickHouseDbType GetDbType() { if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _baseType.GetDbType(); } public IClickHouseTypeInfo GetGenericArgument(int index) { if (_baseType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); if (index != 0) throw new IndexOutOfRangeException(); return _baseType; } private sealed class LowCardinalityColumnReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _baseType; private readonly bool _isNullable; private IClickHouseColumnReader? _baseColumnReader; private int _baseRowCount; private int _keySize; private byte[]? _buffer; private int _position; public LowCardinalityColumnReader(int rowCount, IClickHouseColumnTypeInfo baseType, bool isNullable) { _rowCount = rowCount; _baseType = baseType; _isNullable = isNullable; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return ReadPrefix(sequence); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var result = new SequenceSize(0, 0); var slice = sequence; if (_baseColumnReader == null) { var header = TryReadHeader(slice); if (header == null) return result; _baseRowCount = header.Value.keyCount; _keySize = header.Value.keySize; _baseColumnReader = _baseType.CreateColumnReader(_baseRowCount); slice = slice.Slice(header.Value.bytesRead); result = result.AddBytes(header.Value.bytesRead); } if (_baseRowCount > 0) { var baseResult = _baseColumnReader.ReadNext(slice); _baseRowCount -= baseResult.Elements; slice = slice.Slice(baseResult.Bytes); result = result.AddBytes(baseResult.Bytes); } if (_baseRowCount > 0) return result; if (_buffer == null) { if (slice.Length < sizeof(ulong)) return result; ulong length = 0; slice.Slice(0, sizeof(ulong)).CopyTo(MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref length, 1))); if ((int) length != _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Internal error. Row count check failed: {_rowCount} rows expected, {length} rows detected."); _buffer = new byte[_rowCount * _keySize]; slice = slice.Slice(sizeof(ulong)); result = result.AddBytes(sizeof(ulong)); } var elementCount = Math.Min(_rowCount - _position, (int) (slice.Length / _keySize)); var byteCount = elementCount * _keySize; slice.Slice(0, byteCount).CopyTo(new Span(_buffer, _position * _keySize, byteCount)); _position += elementCount; result += new SequenceSize(byteCount, elementCount); return result; } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { ReadOnlyMemory keys; int keySize; if (_buffer == null) { keys = ReadOnlyMemory.Empty; keySize = 1; } else { keys = new ReadOnlyMemory(_buffer, 0, _position * _keySize); keySize = _keySize; } var valuesColumn = (_baseColumnReader ?? _baseType.CreateColumnReader(0)).EndRead(settings); if (!valuesColumn.TryDipatch(new LowCardinalityTableColumnDispatcher(keys, keySize, _isNullable), out var result)) result = new LowCardinalityTableColumn(keys, keySize, valuesColumn, _isNullable); return result; } public static SequenceSize ReadPrefix(ReadOnlySequence sequence) { if (sequence.Length < sizeof(ulong)) return SequenceSize.Empty; ulong version = 0; sequence.Slice(0, sizeof(ulong)).CopyTo(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref version, 1))); // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeLowCardinality.cpp // Dictionary is written as number N and N keys after them. // Dictionary can be shared for continuous range of granules, so some marks may point to the same position. // Shared dictionary is stored in state and is read once. // // SharedDictionariesWithAdditionalKeys = 1, if (version != 1) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Unexpected dictionary version received: {version}."); return new SequenceSize(sizeof(ulong), 1); } public static (int keySize, int keyCount, int bytesRead)? TryReadHeader(ReadOnlySequence sequence) { Span headerValues = stackalloc ulong[2]; var headerBytes = MemoryMarshal.AsBytes(headerValues); if (sequence.Length < headerBytes.Length) return null; sequence.Slice(0, headerBytes.Length).CopyTo(headerBytes); var keySizeCode = (KeySizeCode)unchecked((byte)headerValues[0]); int keySize; switch (keySizeCode) { case KeySizeCode.UInt8: keySize = 1; break; case KeySizeCode.UInt16: keySize = 2; break; case KeySizeCode.UInt32: keySize = 4; break; case KeySizeCode.UInt64: throw new NotSupportedException("64-bit keys are not supported."); default: throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Internal error. Unexpected size of a key: {keySizeCode}."); } // There are several control flags, but the client always receives 0x2|0x4 // // https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeLowCardinality.cpp // 0x1 Need to read dictionary if it wasn't. // 0x2 Need to read additional keys. Additional keys are stored before indexes as value N and N keys after them. // 0x4 Need to update dictionary. It means that previous granule has different dictionary. var flags = headerValues[0] >> 8; if (flags != (0x2 | 0x4)) throw new NotSupportedException("Received combination of flags is not supported."); var keyCount = checked((int)headerValues[1]); return (keySize, keyCount, headerBytes.Length); } } private sealed class LowCardinalitySkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _baseType; private IClickHouseColumnReaderBase? _baseReader; private int _baseRowCount; private int _keySize; private int _basePosition; private bool _headerSkipped; private int _position; public LowCardinalitySkippingColumnReader(int rowCount, IClickHouseColumnTypeInfo baseType) { _rowCount = rowCount; _baseType = baseType; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return LowCardinalityColumnReader.ReadPrefix(sequence); } public SequenceSize ReadNext(ReadOnlySequence sequence) { var slice = sequence; var result = new SequenceSize(0, 0); if (_baseReader == null) { var header = LowCardinalityColumnReader.TryReadHeader(slice); if (header == null) return result; result = result.AddBytes(header.Value.bytesRead); slice = slice.Slice(header.Value.bytesRead); _baseRowCount = header.Value.keyCount; _keySize = header.Value.keySize; _baseReader = _baseType.CreateSkippingColumnReader(_baseRowCount); } if (_basePosition < _baseRowCount) { var baseResult = _baseReader.ReadNext(slice); _basePosition += baseResult.Elements; result = result.AddBytes(baseResult.Bytes); if (_basePosition < _baseRowCount) return result; } if (!_headerSkipped) { if (sequence.Length - result.Bytes < sizeof(ulong)) return result; ulong length = 0; sequence.Slice(result.Bytes, sizeof(ulong)).CopyTo(MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref length, 1))); if ((int)length != _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Internal error. Row count check failed: {_rowCount} rows expected, {length} rows detected."); result = result.AddBytes(sizeof(ulong)); _headerSkipped = true; } var maxElementsCount = _rowCount - _position; if (maxElementsCount <= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var elementCount = (int)Math.Min((sequence.Length - result.Bytes) / _keySize, maxElementsCount); _position += elementCount; result += new SequenceSize(elementCount * _keySize, elementCount); return result; } } private sealed class LowCardinalityColumnWriter : IClickHouseColumnWriter { private readonly IClickHouseColumnWriter _baseWriter; private readonly IReadOnlyList _indices; private readonly KeySizeCode _keySizeCode; private bool _headerWritten; private int _baseRowCount; private int _position = -1; public string ColumnName => _baseWriter.ColumnName; public string ColumnType { get; } public LowCardinalityColumnWriter(IClickHouseColumnWriter baseWriter, int baseRowCount, string columnType, IReadOnlyList indices, KeySizeCode keySizeCode) { ColumnType = columnType; _indices = indices; _keySizeCode = keySizeCode; _baseWriter = baseWriter; _baseRowCount = baseRowCount; } SequenceSize IClickHouseColumnWriter.WritePrefix(Span writeTo) { if (writeTo.Length < sizeof(ulong)) return SequenceSize.Empty; var writeToVersion = MemoryMarshal.Cast(writeTo.Slice(0, sizeof(ulong))); writeToVersion[0] = 1; // Version return new SequenceSize(sizeof(ulong), 1); } public SequenceSize WriteNext(Span writeTo) { var targetSpan = writeTo; var result = new SequenceSize(0, 0); if (!_headerWritten) { Span headerValues = stackalloc ulong[2]; headerValues[0] = ((0x2 | 0x4) << 8) | (ulong)_keySizeCode; // The size of and element and flags headerValues[1] = (ulong)_baseRowCount; var headerBytes = MemoryMarshal.AsBytes(headerValues); if (headerBytes.Length > targetSpan.Length) return result; headerBytes.CopyTo(targetSpan); targetSpan = targetSpan.Slice(headerBytes.Length); result = result.AddBytes(headerBytes.Length); _headerWritten = true; } if (_baseRowCount > 0) { var baseResult = _baseWriter.WriteNext(targetSpan); _baseRowCount -= baseResult.Elements; result = result.AddBytes(baseResult.Bytes); if (_baseRowCount > 0) return result; if (_baseRowCount != 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Row count check failed: written {-_baseRowCount} more rows then expected."); targetSpan = targetSpan.Slice(baseResult.Bytes); } if (_position < 0) { ulong length = (ulong)_indices.Count; var lengthSpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateReadOnlySpan(ref length, 1)); if (lengthSpan.Length > targetSpan.Length) return result; lengthSpan.CopyTo(targetSpan); result = result.AddBytes(lengthSpan.Length); targetSpan = targetSpan.Slice(lengthSpan.Length); _position = 0; } int size = _indices.Count - _position; int byteSize; switch (_keySizeCode) { case KeySizeCode.UInt8: size = Math.Min(targetSpan.Length, size); byteSize = size; for (int i = 0; i < size; _position++, i++) targetSpan[i] = (byte)_indices[_position]; break; case KeySizeCode.UInt16: size = Math.Min(targetSpan.Length / 2, size); byteSize = size * 2; var ushortSpan = MemoryMarshal.Cast(targetSpan.Slice(0, byteSize)); for (int i = 0; i < size; _position++, i++) ushortSpan[i] = (ushort)_indices[_position]; break; case KeySizeCode.UInt32: size = Math.Min(targetSpan.Length / 4, size); byteSize = size * 4; var intSpan = MemoryMarshal.Cast(targetSpan.Slice(0, byteSize)); for (int i = 0; i < size; _position++, i++) intSpan[i] = _indices[_position]; break; default: throw new InvalidOperationException($"Internal error. Unexpected key size code: {_keySizeCode}."); } result += new SequenceSize(byteSize, size); return result; } } private sealed class LowCardinalityTableColumnDispatcher : IClickHouseTableColumnDispatcher { private readonly ReadOnlyMemory _keys; private readonly int _keySize; private readonly bool _isNullable; public LowCardinalityTableColumnDispatcher(ReadOnlyMemory keys, int keySize, bool isNullable) { _keys = keys; _keySize = keySize; _isNullable = isNullable; } public IClickHouseTableColumn Dispatch(IClickHouseTableColumn column) { return new LowCardinalityTableColumn(_keys, _keySize, column, _isNullable); } } private enum KeySizeCode { UInt8 = 0, UInt16 = 1, UInt32 = 2, UInt64 = 3 } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/MapTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Text.RegularExpressions; namespace Octonica.ClickHouseClient.Types { internal sealed class MapTypeInfo : IClickHouseColumnTypeInfo { // Map(key, value) is the alias for Array(Tuple(key, value)) private readonly KeyValuePair? _typeArgs; private readonly IClickHouseColumnTypeInfo? _underlyingType; public string ComplexTypeName { get; } public string TypeName => "Map"; public int GenericArgumentsCount => _typeArgs == null ? 0 : 2; public MapTypeInfo() { ComplexTypeName = TypeName; } private MapTypeInfo(IClickHouseColumnTypeInfo keyType, IClickHouseColumnTypeInfo valueType, IClickHouseColumnTypeInfo underlyingType) { _typeArgs = new KeyValuePair(keyType, valueType); ComplexTypeName = $"{TypeName}({keyType.ComplexTypeName}, {valueType.ComplexTypeName})"; _underlyingType = underlyingType; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_underlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var fieldType = GetFieldType(); return new MapReader(_underlyingType.CreateColumnReader(rowCount), fieldType); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (_underlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var fieldType = GetFieldType(); return new MapReader(_underlyingType.CreateColumnReader(rowCount, serializationMode), fieldType); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_underlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _underlyingType.CreateSkippingColumnReader(rowCount); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (_underlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _underlyingType.CreateSkippingColumnReader(rowCount, serializationMode); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_underlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var elementType = typeof(T); Type? dictionaryItf = null; foreach (var itf in elementType.GetInterfaces()) { if (itf.IsGenericType && itf.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>)) { if (dictionaryItf != null) { dictionaryItf = null; break; } dictionaryItf = itf; } } IClickHouseColumnWriter underlyingWriter; if (dictionaryItf != null) { var dispatcherType = typeof(DictionaryDispatcher<,>).MakeGenericType(dictionaryItf.GetGenericArguments()); var dispatcher = (IDictionaryDispatcher)Activator.CreateInstance(dispatcherType)!; underlyingWriter = dispatcher.Dispatch(_underlyingType, columnName, rows, columnSettings); } else { underlyingWriter = _underlyingType.CreateColumnWriter(columnName, rows, columnSettings); } return new MapColumnWriter(underlyingWriter); } public IClickHouseParameterWriter CreateParameterWriter() { // TODO: ClickHouseDbType.Map is not supported in DefaultTypeInfoProvider.GetTypeInfo if (_underlyingType == null || _typeArgs == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); Type? dictionaryItf = null; foreach (var itf in type.GetInterfaces()) { if (itf.IsGenericType && itf.GetGenericTypeDefinition() == typeof(IReadOnlyDictionary<,>)) { if (dictionaryItf != null) { dictionaryItf = null; break; } dictionaryItf = itf; } } if (dictionaryItf == null) return _underlyingType.CreateParameterWriter(); var dispatcherTypeArgs = new[] { type }.Concat(dictionaryItf.GetGenericArguments()).ToArray(); var dispatcherType = typeof(ParameterDictionaryDispatcher<,,>).MakeGenericType(dispatcherTypeArgs); var dispatcher = (IDictionaryParameterWirterDispatcher?)Activator.CreateInstance(dispatcherType); Debug.Assert(dispatcher != null); return dispatcher.Dispatch(this); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Map; } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (_typeArgs != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); if (options.Count < 2) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"The type \"{TypeName}\" requires two type arguments: key and value."); if (options.Count > 2) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, $"Too many options for the type \"{TypeName}\"."); var keyType = typeInfoProvider.GetTypeInfo(options[0]); var valueType = typeInfoProvider.GetTypeInfo(options[1]); var underlyingTypeName = $"Array(Tuple({keyType.ComplexTypeName}, {valueType.ComplexTypeName}))"; var undelyingType = typeInfoProvider.GetTypeInfo(underlyingTypeName); return new MapTypeInfo(keyType, valueType, undelyingType); } public Type GetFieldType() { if (_typeArgs == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var keyType = _typeArgs.Value.Key.GetFieldType(); var valueType = _typeArgs.Value.Value.GetFieldType(); var fieldType = typeof(KeyValuePair<,>).MakeGenericType(keyType, valueType).MakeArrayType(); return fieldType; } public IClickHouseTypeInfo GetGenericArgument(int index) { if (_typeArgs == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); switch (index) { case 0: return _typeArgs.Value.Key; case 1: return _typeArgs.Value.Value; default: throw new IndexOutOfRangeException(); } } private sealed class MapReader : IClickHouseColumnReader { private readonly IClickHouseColumnReader _underlyingReader; private readonly Type _fieldType; public MapReader(IClickHouseColumnReader underlyingReader, Type fieldType) { _underlyingReader = underlyingReader; _fieldType = fieldType; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return _underlyingReader.ReadPrefix(sequence); } public SequenceSize ReadNext(ReadOnlySequence sequence) { return _underlyingReader.ReadNext(sequence); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { var column = _underlyingReader.EndRead(settings); var dispatcher = new MapColumnDispatcher(column); return TypeDispatcher.Dispatch(_fieldType, dispatcher); } } private sealed class MapColumnDispatcher : ITypeDispatcher { private readonly IClickHouseTableColumn _column; public MapColumnDispatcher(IClickHouseTableColumn column) { _column = column; } public IClickHouseTableColumn Dispatch() { var column = _column.TryReinterpret(); if (column == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Column was not converted to the type \"{typeof(T)}\" as expected."); return column; } } private sealed class MapColumnWriter : IClickHouseColumnWriter { private readonly IClickHouseColumnWriter _underlyingWriter; public string ColumnName => _underlyingWriter.ColumnName; public string ColumnType { get; } public MapColumnWriter(IClickHouseColumnWriter underlyingWriter) { const string typeNameStart = "Array(Tuple", typeNameEnd = ")"; var typeName = underlyingWriter.ColumnType; if (!typeName.StartsWith(typeNameStart) || !typeName.EndsWith(typeNameEnd)) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. The name of the type \"{typeName}\" doesn't match to the expected pattern \"{Regex.Escape(typeNameStart)}.*{Regex.Escape(typeNameEnd)}\"."); ColumnType = "Map" + typeName[typeNameStart.Length..^typeNameEnd.Length]; _underlyingWriter = underlyingWriter; } SequenceSize IClickHouseColumnWriter.WritePrefix(Span writeTo) { return _underlyingWriter.WritePrefix(writeTo); } public SequenceSize WriteNext(Span writeTo) { return _underlyingWriter.WriteNext(writeTo); } } private interface IDictionaryDispatcher { IClickHouseColumnWriter Dispatch(IClickHouseColumnTypeInfo underlyingType, string columnName, object rows, ClickHouseColumnSettings? columnSettings); } private sealed class DictionaryDispatcher : IDictionaryDispatcher where TKey : notnull { public IClickHouseColumnWriter Dispatch(IClickHouseColumnTypeInfo underlyingType, string columnName, object rows, ClickHouseColumnSettings? columnSettings) { var dictionaryList = (IReadOnlyList>)rows; var listOfLists = new KeyValuePair[dictionaryList.Count][]; for (int i = 0; i < listOfLists.Length; i++) listOfLists[i] = ((IEnumerable>)dictionaryList[i]).ToArray(); return underlyingType.CreateColumnWriter(columnName, listOfLists, columnSettings); } } private interface IDictionaryParameterWirterDispatcher { IClickHouseParameterWriter Dispatch(MapTypeInfo typeInfo); } private sealed class ParameterDictionaryDispatcher : IDictionaryParameterWirterDispatcher where TKey : notnull where TDictionary: IReadOnlyDictionary { public IClickHouseParameterWriter Dispatch(MapTypeInfo typeInfo) { Debug.Assert(typeof(T) == typeof(TDictionary)); Debug.Assert(typeInfo._typeArgs != null); var (keyType, valueType) = typeInfo._typeArgs.Value; var keyWriter = keyType.CreateParameterWriter(); var valueWriter = valueType.CreateParameterWriter(); var writer = new MapParameterWriter(typeInfo, keyWriter, valueWriter); return (IClickHouseParameterWriter)(object)writer; } } private sealed class MapParameterWriter : IClickHouseParameterWriter where TKey : notnull where TDictionary : IReadOnlyDictionary { private readonly MapTypeInfo _mapType; private readonly IClickHouseParameterWriter _keyWriter; private readonly IClickHouseParameterWriter _valueWriter; public MapParameterWriter(MapTypeInfo mapType, IClickHouseParameterWriter keyWriter, IClickHouseParameterWriter valueWriter) { _mapType = mapType; _keyWriter = keyWriter; _valueWriter = valueWriter; } public bool TryCreateParameterValueWriter(TDictionary value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = null; return false; } public StringBuilder Interpolate(StringBuilder queryBuilder, TDictionary dictionary) { queryBuilder.Append("{tuple(["); var valuesBuilder = new StringBuilder(); var isFirst = true; foreach (var (key, value) in dictionary) { if (isFirst) isFirst = false; else queryBuilder.Append(','); _keyWriter.Interpolate(queryBuilder, key); _valueWriter.Interpolate(valuesBuilder, value); } queryBuilder.Append("],["); queryBuilder.Append(valuesBuilder); return queryBuilder.Append("])}"); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { throw new NotImplementedException(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/NothingTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class NothingTableColumn : IClickHouseTableColumn { public int RowCount { get; } public NothingTableColumn(int rowCount) { RowCount = rowCount; } public bool IsNull(int index) { if (index < 0 || index >= RowCount) throw new ArgumentOutOfRangeException(nameof(index)); return true; } public object GetValue(int index) { if (index < 0 || index >= RowCount) throw new ArgumentOutOfRangeException(nameof(index)); return DBNull.Value; } public IClickHouseTableColumn? TryReinterpret() { return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/NothingTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal sealed class NothingTypeInfo : IClickHouseColumnTypeInfo { public string ComplexTypeName => TypeName; public string TypeName => "Nothing"; public int GenericArgumentsCount => 0; public IClickHouseColumnReader CreateColumnReader(int rowCount) { return new NothingColumnReader(rowCount); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new NothingColumnReader(rowCount); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateSkippingColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { return new NothingColumnWriter(columnName, ComplexTypeName, rows.Count); } public IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) return (IClickHouseParameterWriter)(object)NothingParameterWriter.Instance; throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } IClickHouseColumnTypeInfo IClickHouseColumnTypeInfo.GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{TypeName}\" does not support arguments."); } public Type GetFieldType() { return typeof(DBNull); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Nothing; } public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } private sealed class NothingColumnReader : IClickHouseColumnReader { private readonly int _rowCount; private int _position; public NothingColumnReader(int rowCount) { _rowCount = rowCount; } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var size = (int)Math.Min(sequence.Length, _rowCount - _position); _position += size; return new SequenceSize(size, size); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return new NothingTableColumn(_position); } } private sealed class NothingColumnWriter : IClickHouseColumnWriter { private readonly int _count; public string ColumnName { get; } public string ColumnType { get; } private int _position; public NothingColumnWriter(string columnName, string columnType, int count) { ColumnName = columnName; ColumnType = columnType; _count = count; } public SequenceSize WriteNext(Span writeTo) { var size = Math.Min(_count - _position, writeTo.Length); for (int i = 0; i < size; i++) writeTo[i] = 48; // 48 == NOTHING _position += size; return new SequenceSize(size, size); } } internal sealed class NothingParameterWriter : IClickHouseParameterWriter { public static readonly NothingParameterWriter Instance = new NothingParameterWriter(); private NothingParameterWriter() { } public bool TryCreateParameterValueWriter(DBNull value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = EmptyParameterValueWriter.Instance; return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, DBNull value) { return queryBuilder.Append("null"); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { var nothingType = typeInfoProvider.GetTypeInfo("Nothing"); return writeValue(queryBuilder, nothingType, (qb, _) => qb.Append("null")); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/NullableTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Octonica.ClickHouseClient.Exceptions; namespace Octonica.ClickHouseClient.Types { internal sealed class NullableTableColumn : IClickHouseTableColumn { private readonly BitArray? _nullFlags; private readonly IClickHouseTableColumn _baseColumn; public int RowCount => _baseColumn.RowCount; private NullableTableColumn(BitArray? nullFlags, IClickHouseTableColumn baseColumn) { _nullFlags = nullFlags; _baseColumn = baseColumn; } public bool IsNull(int index) { if (_nullFlags != null && _nullFlags[index]) return true; return _baseColumn.IsNull(index); } public object GetValue(int index) { if (_nullFlags != null && _nullFlags[index]) return DBNull.Value; return _baseColumn.GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return TryMakeNullableColumn(_nullFlags, _baseColumn); } public static IClickHouseTableColumn MakeNullableColumn(BitArray? nullFlags, IClickHouseTableColumn baseColumn) { if (!baseColumn.TryDipatch(new NullableTableColumnDispatcher(nullFlags), out var result) || result == null) result = new NullableTableColumn(nullFlags, baseColumn); return result; } public static IClickHouseTableColumn? TryMakeNullableColumn(BitArray? nullFlags, IClickHouseTableColumn notNullableColumn) { return (IClickHouseTableColumn?) TryMakeNullableColumn(typeof(T), nullFlags, notNullableColumn); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return TryMakeNullableArrayColumn(this, _nullFlags, _baseColumn); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } public static IClickHouseArrayTableColumn? TryMakeNullableArrayColumn(IClickHouseTableColumn reinterpretationRoot, BitArray? nullFlags, IClickHouseTableColumn notNullableColumn) { var notNullableArrayColumn = notNullableColumn as IClickHouseArrayTableColumn ?? notNullableColumn.TryReinterpretAsArray(); if (notNullableArrayColumn == null) return null; return new NullableArrayTableColumn(reinterpretationRoot, nullFlags, notNullableArrayColumn); } private static IClickHouseTableColumn? TryMakeNullableColumn(Type underlyingType, BitArray? nullFlags, IClickHouseTableColumn notNullableColumn) { Type dispatcherType; bool reinterpretAsNotNullable = false; if (underlyingType.IsValueType) { var columnType = Nullable.GetUnderlyingType(underlyingType); if (columnType == null) { reinterpretAsNotNullable = true; columnType = underlyingType; } dispatcherType = typeof(NullableStructTableColumnDispatcher<>).MakeGenericType(columnType); } else if (underlyingType.IsClass) { dispatcherType = typeof(NullableObjTableColumnDispatcher<>).MakeGenericType(underlyingType); } else { return null; } var dispatcher = (INullableColumnDispatcher) Activator.CreateInstance(dispatcherType)!; return dispatcher.Dispatch(nullFlags, notNullableColumn, reinterpretAsNotNullable); } private interface INullableColumnDispatcher { IClickHouseTableColumn? Dispatch(BitArray? nullFlags, IClickHouseTableColumn notNullableColumn, bool reinterpretAsNotNullable); } private sealed class NullableStructTableColumnDispatcher : INullableColumnDispatcher where TStruct : struct { public IClickHouseTableColumn? Dispatch(BitArray? nullFlags, IClickHouseTableColumn notNullableColumn, bool reinterpretAsNotNullable) { var reinterpretedColumn = notNullableColumn as IClickHouseTableColumn ?? notNullableColumn.TryReinterpret(); if (reinterpretedColumn == null) return null; var result = new NullableStructTableColumn(nullFlags, reinterpretedColumn); if (!reinterpretAsNotNullable) return result; return result.AsNotNullable(); } } private sealed class NullableObjTableColumnDispatcher : INullableColumnDispatcher where TObj : class { public IClickHouseTableColumn? Dispatch(BitArray? nullFlags, IClickHouseTableColumn notNullableColumn, bool reinterpretAsNotNullable) { Debug.Assert(!reinterpretAsNotNullable); var reinterpretedColumn = notNullableColumn as IClickHouseTableColumn ?? notNullableColumn.TryReinterpret(); if (reinterpretedColumn == null) return null; if (nullFlags == null) return reinterpretedColumn; return new NullableObjTableColumn(nullFlags, reinterpretedColumn); } } private sealed class NullableTableColumnDispatcher : IClickHouseTableColumnDispatcher { private readonly BitArray? _nullFlags; public NullableTableColumnDispatcher(BitArray? nullFlags) { _nullFlags = nullFlags; } public IClickHouseTableColumn? Dispatch(IClickHouseTableColumn column) { Type type = typeof(T); Type dispatcherType; if (type.IsValueType) { var columnType = Nullable.GetUnderlyingType(type) ?? type; dispatcherType = typeof(NullableStructTableColumnDispatcher<>).MakeGenericType(columnType); } else if (type.IsClass) { dispatcherType = typeof(NullableObjTableColumnDispatcher<>).MakeGenericType(type); } else { return null; } var dispatcher = (INullableColumnDispatcher)Activator.CreateInstance(dispatcherType)!; return dispatcher.Dispatch(_nullFlags, column, false); } } } internal sealed class NullableStructTableColumn : IClickHouseTableColumn where TStruct : struct { private readonly BitArray? _nullFlags; private readonly IClickHouseTableColumn _baseColumn; public int RowCount => _baseColumn.RowCount; public TStruct? DefaultValue => null; public NullableStructTableColumn(BitArray? nullFlags, IClickHouseTableColumn baseColumn) { _nullFlags = nullFlags; _baseColumn = baseColumn; } public bool IsNull(int index) { if (_nullFlags != null && _nullFlags[index]) return true; return _baseColumn.IsNull(index); } public TStruct? GetValue(int index) { if (_nullFlags != null && _nullFlags[index]) return null; return _baseColumn.GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return NullableTableColumn.TryMakeNullableColumn(_nullFlags, _baseColumn); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return NullableTableColumn.TryMakeNullableArrayColumn(this, _nullFlags, _baseColumn); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { return (object?) GetValue(index) ?? DBNull.Value; } public IClickHouseTableColumn AsNotNullable() { if (_nullFlags == null) return _baseColumn; return new NullableStructTableColumnNotNullableAdapter(_nullFlags, _baseColumn); } } internal sealed class NullableStructTableColumnNotNullableAdapter : IClickHouseTableColumn where TStruct : struct { private readonly BitArray _nullFlags; private readonly IClickHouseTableColumn _baseColumn; public int RowCount => _baseColumn.RowCount; public TStruct DefaultValue => _baseColumn.DefaultValue; public NullableStructTableColumnNotNullableAdapter(BitArray nullFlags, IClickHouseTableColumn baseColumn) { _nullFlags = nullFlags; _baseColumn = baseColumn; } public bool IsNull(int index) { return _nullFlags[index] || _baseColumn.IsNull(index); } public TStruct GetValue(int index) { if (_nullFlags[index]) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, $"Can't convert NULL to \"{typeof(TStruct)}\"."); return _baseColumn.GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return NullableTableColumn.TryMakeNullableColumn(_nullFlags, _baseColumn); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return NullableTableColumn.TryMakeNullableArrayColumn(this, _nullFlags, _baseColumn); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { if (_nullFlags != null && _nullFlags[index]) return DBNull.Value; return _baseColumn.GetValue(index); } public NullableStructTableColumn Unguard() { return new NullableStructTableColumn(_nullFlags, _baseColumn); } } internal sealed class NullableObjTableColumn : IClickHouseTableColumn where TObj : class { private readonly BitArray _nullFlags; private readonly IClickHouseTableColumn _baseColumn; public int RowCount => _baseColumn.RowCount; public TObj? DefaultValue => null; public NullableObjTableColumn(BitArray nullFlags, IClickHouseTableColumn baseColumn) { _nullFlags = nullFlags; _baseColumn = baseColumn; } public bool IsNull(int index) { return _nullFlags[index] || _baseColumn.IsNull(index); } public TObj? GetValue(int index) { if (_nullFlags[index]) return null; return _baseColumn.GetValue(index); } public NullableObjTableColumn ReinterpretAsObj(Func convert) where TRes : class { IClickHouseTableColumn updColumn; if (_baseColumn is IClickHouseReinterpretedTableColumn baseReinterpreted) updColumn = baseReinterpreted.Chain(convert); else updColumn = new ReinterpretedTableColumn(_baseColumn, convert); return new NullableObjTableColumn(_nullFlags, updColumn); } public NullableStructTableColumn ReinterpretAsStruct(Func convert) where TRes : struct { IClickHouseTableColumn updColumn; if (_baseColumn is IClickHouseReinterpretedTableColumn baseReinterpreted) updColumn = baseReinterpreted.Chain(convert); else updColumn = new ReinterpretedTableColumn(_baseColumn, convert); return new NullableStructTableColumn(_nullFlags, updColumn); } public IClickHouseTableColumn? TryReinterpret() { return NullableTableColumn.TryMakeNullableColumn(_nullFlags, _baseColumn); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return NullableTableColumn.TryMakeNullableArrayColumn(this, _nullFlags, _baseColumn); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { return (object?) GetValue(index) ?? DBNull.Value; } } internal sealed class NullableArrayTableColumn : IClickHouseArrayTableColumn { private readonly IClickHouseTableColumn _reinterpretationRoot; private readonly BitArray? _nullFlags; private readonly IClickHouseArrayTableColumn _arrayColumn; public int RowCount => throw new NotImplementedException(); public NullableArrayTableColumn(IClickHouseTableColumn reinterpretationRoot, BitArray? nullFlags, IClickHouseArrayTableColumn arrayColumn) { _reinterpretationRoot = reinterpretationRoot; _nullFlags = nullFlags; _arrayColumn = arrayColumn; } public object GetValue(int index) { if (_nullFlags != null && _nullFlags[index]) return DBNull.Value; return _arrayColumn.GetValue(index); } public bool IsNull(int index) { return (_nullFlags != null && _nullFlags[index]) || _arrayColumn.IsNull(index); } public IClickHouseTableColumn? TryReinterpret() { return _reinterpretationRoot as IClickHouseTableColumn ?? _reinterpretationRoot.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return _reinterpretationRoot as IClickHouseArrayTableColumn ?? _reinterpretationRoot.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } public int CopyTo(int index, Span buffer, int dataOffset) { if (_nullFlags != null && _nullFlags[index]) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Can't copy NULL value to the buffer."); return _arrayColumn.CopyTo(index, buffer, dataOffset); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/NullableTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class NullableTypeInfo : IClickHouseColumnTypeInfo { public string ComplexTypeName { get; } public string TypeName => "Nullable"; public int GenericArgumentsCount => UnderlyingType == null ? 0 : 1; public IClickHouseColumnTypeInfo? UnderlyingType { get; } public NullableTypeInfo() { ComplexTypeName = TypeName; } public NullableTypeInfo(IClickHouseColumnTypeInfo underlyingType) { if (underlyingType is NullableTypeInfo) throw new ArgumentException("The underlying type can't be nullable.", nameof(underlyingType)); UnderlyingType = underlyingType ?? throw new ArgumentNullException(nameof(underlyingType)); ComplexTypeName = $"{TypeName}({UnderlyingType.ComplexTypeName})"; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new NullableColumnReader(rowCount, UnderlyingType); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new NullableSkippingColumnReader(rowCount, UnderlyingType); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateSkippingColumnReader(rowCount); throw new NotSupportedException($"Custom serialization for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new NullableColumnWriter(columnName, rows, columnSettings, UnderlyingType); } public IClickHouseParameterWriter CreateParameterWriter() { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var type = typeof(T); if (type == typeof(DBNull)) return (IClickHouseParameterWriter)(object)new NullableParameterWriter(this, NothingTypeInfo.NothingParameterWriter.Instance); if (type.IsValueType && type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>)) { var valueType = type.GetGenericArguments()[0]; var dispatcherType = typeof(NullableStructParameterWriterDispatcher<>).MakeGenericType(valueType); var dispatcher = (INullableParameterWriterDispatcher?)Activator.CreateInstance(dispatcherType); Debug.Assert(dispatcher != null); return dispatcher.Dispatch(this); } // The type is either a non-value or a non-nullable stucture. In both cases it can be interpreted as non-nullable. var underlyingWriter = UnderlyingType.CreateParameterWriter(); return new NullableParameterWriter(this, underlyingWriter); } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (UnderlyingType != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); if (options.Count > 1) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"Too many arguments in the definition of \"{TypeName}\"."); var underlyingType = typeInfoProvider.GetTypeInfo(options[0]); return new NullableTypeInfo(underlyingType); } public Type GetFieldType() { if (UnderlyingType == null) return typeof(DBNull); var underlyingFieldType = UnderlyingType.GetFieldType(); if (underlyingFieldType.IsValueType) return typeof(Nullable<>).MakeGenericType(underlyingFieldType); return underlyingFieldType; } public ClickHouseDbType GetDbType() { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return UnderlyingType.GetDbType(); } public IClickHouseTypeInfo GetGenericArgument(int index) { if (UnderlyingType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); if (index != 0) throw new IndexOutOfRangeException(); return UnderlyingType; } private sealed class NullableColumnReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _underlyingType; private BitArray? _nullFlags; private IClickHouseColumnReader? _baseColumnReader; private int _nullFlagPosition; public NullableColumnReader(int rowCount, IClickHouseColumnTypeInfo underlyingType) { _rowCount = rowCount; _underlyingType = underlyingType; } public SequenceSize ReadNext(ReadOnlySequence sequence) { int bytesCount = 0; if (_baseColumnReader == null) { foreach (var mem in sequence) { if (_nullFlagPosition == _rowCount) break; foreach (var byteVal in mem.Span) { if (byteVal != 0) { if (_nullFlags == null) _nullFlags = new BitArray(_rowCount, false); _nullFlags[_nullFlagPosition] = true; } bytesCount++; if (_rowCount == ++_nullFlagPosition) break; } } if (_nullFlagPosition < _rowCount) return new SequenceSize(bytesCount, 0); _baseColumnReader = _underlyingType.CreateColumnReader(_rowCount); } var baseResult = _baseColumnReader.ReadNext(sequence.Slice(bytesCount)); return new SequenceSize(bytesCount + baseResult.Bytes, baseResult.Elements); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { var baseReader = _baseColumnReader ?? _underlyingType.CreateColumnReader(0); return NullableTableColumn.MakeNullableColumn(_nullFlags, baseReader.EndRead(settings)); } } private sealed class NullableSkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _rowCount; private readonly IClickHouseColumnTypeInfo _underlyingType; private IClickHouseColumnReaderBase? _baseReader; private int _position; public NullableSkippingColumnReader(int rowCount, IClickHouseColumnTypeInfo underlyingType) { _rowCount = rowCount; _underlyingType = underlyingType; } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_baseReader != null) { var result = _baseReader.ReadNext(sequence); return result; } var prefixBytesCount = _rowCount - _position; if (prefixBytesCount < 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); if (sequence.Length <= prefixBytesCount) { _position += (int)sequence.Length; return new SequenceSize((int)sequence.Length, 0); } else { _position += prefixBytesCount; } _baseReader = _underlyingType.CreateSkippingColumnReader(_rowCount); var baseSize = _baseReader.ReadNext(sequence.Slice(prefixBytesCount)); return new SequenceSize(baseSize.Bytes + prefixBytesCount, baseSize.Elements); } } private sealed class NullableColumnWriter : IClickHouseColumnWriter { private readonly IReadOnlyList _rows; private readonly IClickHouseColumnWriter _internalColumnWriter; public string ColumnName { get; } public string ColumnType { get; } private int _position; public NullableColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo underlyingTypeInfo) { if (underlyingTypeInfo == null) throw new ArgumentNullException(nameof(underlyingTypeInfo)); _rows = rows ?? throw new ArgumentNullException(nameof(rows)); ColumnName = columnName ?? throw new ArgumentNullException(nameof(columnName)); if (typeof(T).IsValueType && typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(Nullable<>)) { var valueType = typeof(T).GetGenericArguments()[0]; var dispatcherType = typeof(ValueOrDefaultListDispatcher<>).MakeGenericType(valueType); var columnDispatcher = (IValueOrDefaultListDispatcherBase) Activator.CreateInstance(dispatcherType)!; _internalColumnWriter = columnDispatcher.Dispatch(columnName, rows, columnSettings, underlyingTypeInfo); } else { _internalColumnWriter = underlyingTypeInfo.CreateColumnWriter(columnName, rows, columnSettings); } ColumnType = $"Nullable({_internalColumnWriter.ColumnType})"; } public SequenceSize WriteNext(Span writeTo) { if (_position == _rows.Count) return _internalColumnWriter.WriteNext(writeTo); var len = Math.Min(_rows.Count - _position, writeTo.Length); for (int i = 0; i < len; i++) { writeTo[i] = _rows[_position + i] == null ? (byte) 1 : (byte) 0; } _position += len; if (_position < _rows.Count) return new SequenceSize(len, 0); var size = _internalColumnWriter.WriteNext(writeTo.Slice(len)); return new SequenceSize(size.Bytes + len, size.Elements); } } private interface IValueOrDefaultListDispatcherBase { IClickHouseColumnWriter Dispatch(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo underlyingTypeInfo); } private sealed class ValueOrDefaultListDispatcher : IValueOrDefaultListDispatcherBase where TValue : struct { public IClickHouseColumnWriter Dispatch(string columnName, object rows, ClickHouseColumnSettings? columnSettings, IClickHouseColumnTypeInfo underlyingTypeInfo) { var genericList = (IReadOnlyList) rows; var listWrapper = MappedReadOnlyList.Map(genericList, item => item ?? default); return underlyingTypeInfo.CreateColumnWriter(columnName, listWrapper, columnSettings); } } private interface INullableParameterWriterDispatcher { IClickHouseParameterWriter Dispatch(NullableTypeInfo typeInfo); } private sealed class NullableStructParameterWriterDispatcher : INullableParameterWriterDispatcher where T : struct { public IClickHouseParameterWriter Dispatch(NullableTypeInfo typeInfo) { Debug.Assert(typeInfo.UnderlyingType != null); var underlyingWriter = typeInfo.UnderlyingType.CreateParameterWriter(); return new NullableStructParameterWriter(typeInfo, underlyingWriter); } } private sealed class NullableStructParameterWriter : IClickHouseParameterWriter where T : struct { private readonly NullableTypeInfo _typeIfno; private readonly IClickHouseParameterWriter _underlyingWritrer; public NullableStructParameterWriter(NullableTypeInfo typeIfno, IClickHouseParameterWriter underlyingWritrer) { _typeIfno = typeIfno; _underlyingWritrer = underlyingWritrer; } public bool TryCreateParameterValueWriter(T? value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { if (value == null) { valueWriter = new SimpleLiteralValueWriter("null".AsMemory()); return true; } return _underlyingWritrer.TryCreateParameterValueWriter(value.Value, isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, T? value) { if (value != null) { queryBuilder.Append("CAST(("); _underlyingWritrer.Interpolate(queryBuilder, value.Value); queryBuilder.Append(") AS ").Append(_typeIfno.ComplexTypeName).Append(')'); } else { queryBuilder.Append("null::").Append(_typeIfno.ComplexTypeName); } return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _underlyingWritrer.Interpolate(queryBuilder, typeInfoProvider, (qb, typeInfo, writeElement) => { Debug.Assert(_typeIfno.UnderlyingType != null); if (typeInfo.ComplexTypeName == _typeIfno.UnderlyingType.ComplexTypeName) { // The value of the type Nullable(Nothing) can't be passed as a parameter if (typeInfo.ComplexTypeName != "Nothing") return writeValue(qb, _typeIfno, FunctionHelper.Apply); } var nullableTypeInfo = typeInfo; if (nullableTypeInfo.TypeName != "Nullable") nullableTypeInfo = new NullableTypeInfo(nullableTypeInfo); return writeValue(qb, nullableTypeInfo, (qb2, realWrite) => { qb2.Append("CAST(("); writeElement(qb2, realWrite); qb2.Append(") AS ").Append(_typeIfno.ComplexTypeName).Append(')'); return qb2; }); }); } } private sealed class NullableParameterWriter : IClickHouseParameterWriter { private readonly NullableTypeInfo _typeIfno; private readonly IClickHouseParameterWriter _underlyingWritrer; public NullableParameterWriter(NullableTypeInfo typeIfno, IClickHouseParameterWriter underlyingWritrer) { _typeIfno = typeIfno; _underlyingWritrer = underlyingWritrer; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { if (value == null) { valueWriter = new SimpleLiteralValueWriter("null".AsMemory()); return true; } return _underlyingWritrer.TryCreateParameterValueWriter(value, isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { if (value != null) { queryBuilder.Append("CAST(("); _underlyingWritrer.Interpolate(queryBuilder, value); queryBuilder.Append(") AS ").Append(_typeIfno.ComplexTypeName).Append(')'); } else { queryBuilder.Append("null::").Append(_typeIfno.ComplexTypeName); } return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _underlyingWritrer.Interpolate(queryBuilder, typeInfoProvider, (qb, typeInfo, writeElement) => { Debug.Assert(_typeIfno.UnderlyingType != null); if (typeInfo.ComplexTypeName == _typeIfno.UnderlyingType.ComplexTypeName) { // The value of the type Nullable(Nothing) can't be passed as a parameter if (typeInfo.ComplexTypeName != "Nothing") return writeValue(qb, _typeIfno, FunctionHelper.Apply); } var nullableTypeInfo = typeInfo; if (nullableTypeInfo.TypeName != "Nullable") nullableTypeInfo = new NullableTypeInfo(nullableTypeInfo); return writeValue(qb, nullableTypeInfo, (qb2, realWrite)=> { qb2.Append("CAST(("); writeElement(qb2, realWrite); qb2.Append(") AS ").Append(_typeIfno.ComplexTypeName).Append(')'); return qb2; }); }); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ObjectColumnAdapter.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class ObjectColumnAdapter : IClickHouseReinterpretedTableColumn { private readonly IClickHouseTableColumn _tableColumn; public int RowCount => _tableColumn.RowCount; public object DefaultValue => throw new NotSupportedException("The default value is not supported for the column of type Object."); public ObjectColumnAdapter(IClickHouseTableColumn tableColumn) { _tableColumn = tableColumn; } public bool IsNull(int index) { return _tableColumn.IsNull(index); } public object GetValue(int index) { return _tableColumn.GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return _tableColumn.TryReinterpret(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseReinterpretedTableColumn Chain(Func reinterpret) { return new ReinterpretedObjectTableColumn(_tableColumn, reinterpret); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ReinterpretedArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class ReinterpretedArrayTableColumn : IClickHouseArrayTableColumn { private readonly IClickHouseTableColumn _reinterpretationRoot; private readonly IClickHouseArrayTableColumn _arrayColumn; public int RowCount => _arrayColumn.RowCount; public ReinterpretedArrayTableColumn(IClickHouseTableColumn reinterpretationRoot, IClickHouseArrayTableColumn arrayColumn) { _reinterpretationRoot = reinterpretationRoot; _arrayColumn = arrayColumn; } public int CopyTo(int index, Span buffer, int dataOffset) { return _arrayColumn.CopyTo(index, buffer, dataOffset); } public object GetValue(int index) { return _arrayColumn.GetValue(index); } public bool IsNull(int index) { return _arrayColumn.IsNull(index); } public IClickHouseTableColumn? TryReinterpret() { return _reinterpretationRoot as IClickHouseTableColumn ?? _reinterpretationRoot.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return _reinterpretationRoot as IClickHouseArrayTableColumn ?? _reinterpretationRoot.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ReinterpretedObjectTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class ReinterpretedObjectTableColumn : IClickHouseReinterpretedTableColumn { private readonly IClickHouseTableColumn _column; private readonly Func _reinterpret; public int RowCount => _column.RowCount; TRes IClickHouseTableColumn.DefaultValue => throw new NotSupportedException("The default value is not supported for the column of type Object."); public ReinterpretedObjectTableColumn(IClickHouseTableColumn column, Func reinterpret) { _column = column ?? throw new ArgumentNullException(nameof(column)); _reinterpret = reinterpret ?? throw new ArgumentNullException(nameof(reinterpret)); } public bool IsNull(int index) { return _column.IsNull(index); } public TRes GetValue(int index) { var value = _column.GetValue(index); var converted = _reinterpret(value); return converted; } object IClickHouseTableColumn.GetValue(int index) { var value = _column.GetValue(index); var converted = _reinterpret(value); if (converted is null) return DBNull.Value; return converted; } public IClickHouseTableColumn? TryReinterpret() { return (_column as IClickHouseTableColumn) ?? _column.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return (_column as IClickHouseArrayTableColumn) ?? _column.TryReinterpretAsArray(); } public bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } public IClickHouseReinterpretedTableColumn Chain(Func convert) { return new ReinterpretedObjectTableColumn(_column, FunctionHelper.Combine(_reinterpret, convert)); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/ReinterpretedTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class ReinterpretedTableColumn : IClickHouseReinterpretedTableColumn { private readonly IClickHouseTableColumn? _reinterpretationRoot; private readonly IClickHouseTableColumn _sourceColumn; private readonly Func _reinterpret; public int RowCount => _sourceColumn.RowCount; public TTo DefaultValue { get; } public ReinterpretedTableColumn(IClickHouseTableColumn sourceColumn, Func reinterpret) : this(null, sourceColumn, reinterpret) { } public ReinterpretedTableColumn(IClickHouseTableColumn? reinterpretationRoot, IClickHouseTableColumn sourceColumn, Func reinterpret) { _reinterpretationRoot = reinterpretationRoot; _sourceColumn = sourceColumn ?? throw new ArgumentNullException(nameof(sourceColumn)); _reinterpret = reinterpret ?? throw new ArgumentNullException(nameof(reinterpret)); DefaultValue = _reinterpret(_sourceColumn.DefaultValue); } public bool IsNull(int index) { return _sourceColumn.IsNull(index); } public TTo GetValue(int index) { var value = _sourceColumn.GetValue(index); return _reinterpret(value); } public IClickHouseTableColumn? TryReinterpret() { var reinterpretationRoot = _reinterpretationRoot ?? _sourceColumn; return (reinterpretationRoot as IClickHouseTableColumn) ?? reinterpretationRoot.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { var reinterpretationRoot = _reinterpretationRoot ?? _sourceColumn; return (reinterpretationRoot as IClickHouseArrayTableColumn) ?? reinterpretationRoot.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } object IClickHouseTableColumn.GetValue(int index) { if (_sourceColumn.IsNull(index)) return DBNull.Value; var sourceValue = _sourceColumn.GetValue(index); var reinterpreted = _reinterpret(sourceValue); if (reinterpreted is null) return DBNull.Value; return reinterpreted; } public IClickHouseReinterpretedTableColumn Chain(Func reinterpret) { return new ReinterpretedTableColumn(_reinterpretationRoot, _sourceColumn, FunctionHelper.Combine(_reinterpret, reinterpret)); } } internal sealed class ReinterpretedTableColumn : IClickHouseReinterpretedTableColumn { private readonly IClickHouseTableColumn _reinterpretationRoot; private readonly IClickHouseTableColumn _column; public int RowCount => _column.RowCount; public TValue DefaultValue => _column.DefaultValue; public ReinterpretedTableColumn(IClickHouseTableColumn reinterpretationRoot, IClickHouseTableColumn column) { _reinterpretationRoot = reinterpretationRoot ?? throw new ArgumentNullException(nameof(reinterpretationRoot)); _column = column ?? throw new ArgumentNullException(nameof(column)); } public bool IsNull(int index) { return _column.IsNull(index); } public TValue GetValue(int index) { return _column.GetValue(index); } object IClickHouseTableColumn.GetValue(int index) { return ((IClickHouseTableColumn) _column).GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { return (_reinterpretationRoot as IClickHouseTableColumn) ?? _reinterpretationRoot.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return (_reinterpretationRoot as IClickHouseArrayTableColumn) ?? _reinterpretationRoot.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } public IClickHouseReinterpretedTableColumn Chain(Func reinterpret) { return new ReinterpretedTableColumn(_reinterpretationRoot, _column, reinterpret); } } internal sealed class ReinterpretedTableColumn : IClickHouseTableColumn { private readonly IClickHouseTableColumn _column; private readonly Func _convertValue; public int RowCount => _column.RowCount; public ReinterpretedTableColumn(IClickHouseTableColumn column, Func convertValue) { _column = column ?? throw new ArgumentNullException(nameof(column)); _convertValue = convertValue ?? throw new ArgumentNullException(nameof(convertValue)); } public bool IsNull(int index) { return _column.IsNull(index); } public object GetValue(int index) { var value = _column.GetValue(index); return _convertValue(value); } public IClickHouseTableColumn? TryReinterpret() { return (_column as IClickHouseTableColumn) ?? _column.TryReinterpret(); } IClickHouseArrayTableColumn? IClickHouseTableColumn.TryReinterpretAsArray() { return (_column as IClickHouseArrayTableColumn) ?? _column.TryReinterpretAsArray(); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } public static IClickHouseTableColumn GetReinterpetedTableColumn(IClickHouseTableColumn column, Type targetType, Func fallbackConvertValue) { return GetReinterpetedTableColumn(column, TypeDispatcher.Create(targetType), fallbackConvertValue); } internal static IClickHouseTableColumn GetReinterpetedTableColumn(IClickHouseTableColumn column, ITypeDispatcher typeDispatcher, Func fallbackConvertValue) { return typeDispatcher.Dispatch(new ReinterpretedTableColumnDispatcher(column, fallbackConvertValue)); } private sealed class ReinterpretedTableColumnDispatcher : ITypeDispatcher { private readonly IClickHouseTableColumn _column; private readonly Func _fallbackConvertValue; public ReinterpretedTableColumnDispatcher(IClickHouseTableColumn column, Func fallbackConvertValue) { _column = column; _fallbackConvertValue = fallbackConvertValue; } public IClickHouseTableColumn Dispatch() { var reinterpretedColumn = (_column as IClickHouseTableColumn) ?? _column.TryReinterpret(); if (reinterpretedColumn != null) return reinterpretedColumn; return new ReinterpretedTableColumn(_column, _fallbackConvertValue); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/SimpleLiteralValueWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using System; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class SimpleLiteralValueWriter : IClickHouseParameterValueWriter { private readonly ReadOnlyMemory _value; public int Length => Encoding.UTF8.GetByteCount(_value.Span); public SimpleLiteralValueWriter(ReadOnlyMemory value) { _value = value; } public int Write(Memory buffer) { return Encoding.UTF8.GetBytes(_value.Span, buffer.Span); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/SimpleParameterWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class SimpleParameterWriter : IClickHouseParameterWriter where T : IFormattable { private readonly string? _valueType; private readonly IClickHouseColumnTypeInfo _type; private readonly string? _format; private readonly bool _appendTypeCast; public SimpleParameterWriter(IClickHouseColumnTypeInfo type, string? format = null, bool appendTypeCast = false) : this(null, type, format, appendTypeCast) { } public SimpleParameterWriter(string? valueType, IClickHouseColumnTypeInfo type, string? format = null, bool appendTypeCast = false) { _valueType = valueType; _type = type; _format = format; _appendTypeCast = appendTypeCast; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = value.ToString(_format, CultureInfo.InvariantCulture); valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { var strVal = value.ToString(_format, CultureInfo.InvariantCulture); queryBuilder.Append(strVal); if (_appendTypeCast) queryBuilder.Append("::").Append(_type.ComplexTypeName); return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { if (_valueType == null) return writeValue(queryBuilder, _type, FunctionHelper.Apply); var valueTypeInfo = typeInfoProvider.GetTypeInfo(_valueType); if (_valueType != _type.ComplexTypeName) return writeValue(queryBuilder, valueTypeInfo, (qb, realWrite) => realWrite(qb).Append("::").Append(_type.ComplexTypeName)); return writeValue(queryBuilder, valueTypeInfo, FunctionHelper.Apply); } } internal sealed class SimpleParameterWriter : IClickHouseParameterWriter where TOut : IFormattable { private readonly string? _valueType; private readonly IClickHouseColumnTypeInfo _type; private readonly Func _convert; private readonly string? _format; private readonly bool _appendTypeCast; public SimpleParameterWriter(IClickHouseColumnTypeInfo type, Func convert) : this(null, type, null, false, convert) { } public SimpleParameterWriter(IClickHouseColumnTypeInfo type, bool appendTypeCast, Func convert) : this(null, type, null, appendTypeCast, convert) { } public SimpleParameterWriter(IClickHouseColumnTypeInfo type, string? format, Func convert) : this(null, type, format, false, convert) { } public SimpleParameterWriter(string? valueType, IClickHouseColumnTypeInfo type, string? format, bool appendTypeCast, Func convert) { _valueType = valueType; _type = type; _format = format; _appendTypeCast = appendTypeCast; _convert = convert; } public bool TryCreateParameterValueWriter(TIn value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { var strVal = _convert(value).ToString(_format, CultureInfo.InvariantCulture); valueWriter = new SimpleLiteralValueWriter(strVal.AsMemory()); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, TIn value) { var strVal = _convert(value).ToString(_format, CultureInfo.InvariantCulture); queryBuilder.Append(strVal); if (_valueType != null) queryBuilder.Append("::").Append(_valueType); if (_appendTypeCast) { if (_valueType == null || _valueType != _type.ComplexTypeName) queryBuilder.Append("::").Append(_type.ComplexTypeName); } return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { if (_valueType == null) return writeValue(queryBuilder, _type, FunctionHelper.Apply); var valueTypeInfo = typeInfoProvider.GetTypeInfo(_valueType); if (_valueType != _type.ComplexTypeName) return writeValue(queryBuilder, valueTypeInfo, (qb, realWrite) => realWrite(qb).Append("::").Append(_type.ComplexTypeName)); return writeValue(queryBuilder, valueTypeInfo, FunctionHelper.Apply); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/SimpleSkippingColumnReader.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using System; using System.Buffers; namespace Octonica.ClickHouseClient.Types { internal sealed class SimpleSkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _elementSize; private readonly int _rowCount; private int _position; public SimpleSkippingColumnReader(int elementSize, int rowCount) { _elementSize = elementSize; _rowCount = rowCount; } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var elementCount = (int)Math.Min(_rowCount - _position, sequence.Length / _elementSize); var byteCount = elementCount * _elementSize; _position += elementCount; return new SequenceSize(byteCount, elementCount); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/SimpleTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { /// /// Represents the base class for types that has no arguments. /// public abstract class SimpleTypeInfo : IClickHouseColumnTypeInfo { /// public string ComplexTypeName => TypeName; /// public string TypeName { get; } /// /// Gets the number of generic arguments in the list of arguments. /// /// Always returns 0. public int GenericArgumentsCount => 0; /// /// Initializes a new instance of with the specified name. /// /// The name of the type protected SimpleTypeInfo(string typeName) { TypeName = typeName ?? throw new ArgumentNullException(nameof(typeName)); } /// public abstract IClickHouseColumnReader CreateColumnReader(int rowCount); /// public abstract IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount); /// public abstract IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings); /// public abstract IClickHouseParameterWriter CreateParameterWriter(); /// public abstract Type GetFieldType(); /// public abstract ClickHouseDbType GetDbType(); /// /// Gets the generic arguments at the specified position. /// /// The zero-based index of the generic argument. /// Always throws . public IClickHouseTypeInfo GetGenericArgument(int index) { throw new NotSupportedException($"The type \"{TypeName}\" doesn't have generic arguments."); } IClickHouseColumnTypeInfo IClickHouseColumnTypeInfo.GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{TypeName}\" does not support arguments."); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/SparseColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal sealed class SparseColumn : IClickHouseTableColumn { private readonly IClickHouseTableColumn _valuesColumn; private readonly List _offsets; private readonly bool _trailingDefaults; private int _lastHit = 0; public int RowCount { get; } public T DefaultValue { get; } public SparseColumn(IClickHouseTableColumn valuesColumn, int rowCount, List offsets, bool trailingDefaults) { _valuesColumn = valuesColumn; RowCount = rowCount; _offsets = offsets; _trailingDefaults = trailingDefaults; DefaultValue = _valuesColumn.DefaultValue; } private SparseColumn(IClickHouseTableColumn valuesColumn, int rowCount, List offsets, bool trailingDefaults, int lastHit) { _valuesColumn = valuesColumn; RowCount = rowCount; _offsets = offsets; _trailingDefaults = trailingDefaults; _lastHit = lastHit; DefaultValue = _valuesColumn.DefaultValue; } public T GetValue(int index) { var valueIndex = GetValueIndex(index); if (valueIndex < 0) return DefaultValue; return _valuesColumn.GetValue(valueIndex); } public bool IsNull(int index) { var valueIndex = GetValueIndex(index); if (valueIndex < 0) return DefaultValue is null; return _valuesColumn.IsNull(valueIndex); } public bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out TOut dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } public IClickHouseTableColumn? TryReinterpret() { var valuesReinterpreted = _valuesColumn.TryReinterpret(); if (valuesReinterpreted == null) return null; return new SparseColumn(valuesReinterpreted, RowCount, _offsets, _trailingDefaults, _lastHit); } object IClickHouseTableColumn.GetValue(int index) { return (object?)GetValue(index) ?? DBNull.Value; } private int GetValueIndex(int index) { if (index < 0 || index >= RowCount) throw new ArgumentOutOfRangeException(nameof(index)); if (_offsets.Count == 0) return -1; var lastHit = _lastHit; var lastHitIdx = _offsets[lastHit]; if (index > lastHitIdx) { var next = lastHit + 1; if (next == _offsets.Count) { if (_trailingDefaults) return -1; return _offsets.Count + (index - lastHitIdx); } var nextIdx = _offsets[next]; if (index < nextIdx) return -1; // The most expected case if (index == nextIdx) { _lastHit = next; return next; } } else if (index == lastHitIdx) { return lastHit; } else if (lastHit == 0) { // index < lastHitIdx return -1; } lastHit = _offsets.BinarySearch(index); if (lastHit >= 0) { _lastHit = lastHit; return lastHit; } lastHit = ~lastHit; _lastHit = Math.Max(lastHit - 1, 0); if (lastHit < _offsets.Count || _trailingDefaults) return -1; return _offsets.Count + (index - _offsets[^1]); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringByteArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class StringByteArrayTableColumn : StringTableColumnBase { public override byte[] DefaultValue => Array.Empty(); public StringByteArrayTableColumn(Encoding encoding, List<(int segmentIndex, int offset, int length)> layouts, List> segments) : base(encoding, layouts, segments) { } protected override byte[] GetValue(Encoding encoding, ReadOnlySpan span) { if (span.IsEmpty) return Array.Empty(); var result = new byte[span.Length]; span.CopyTo(result); return result; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringCharArrayTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class StringCharArrayTableColumn : StringTableColumnBase { public override char[] DefaultValue => Array.Empty(); public StringCharArrayTableColumn(Encoding encoding, List<(int segmentIndex, int offset, int length)> layouts, List> segments) : base(encoding, layouts, segments) { } protected override char[] GetValue(Encoding encoding, ReadOnlySpan span) { if (span.IsEmpty) return Array.Empty(); var charCount = encoding.GetCharCount(span); var result = new char[charCount]; var length = encoding.GetChars(span, result); Debug.Assert(length == charCount); return result; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringLiteralValueWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using System; using System.Collections.Generic; using System.Diagnostics; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class StringLiteralValueWriter : IClickHouseParameterValueWriter { private readonly ReadOnlyMemory _value; private readonly bool _includeQuotes; private readonly List? _escapeIndices; public int Length { get; } public StringLiteralValueWriter(ReadOnlyMemory value, bool includeQuotes) { var encoding = Encoding.UTF8; int length = includeQuotes ? 4 : 0, i = 0; while (i < value.Length) { var idx = value.Span.Slice(i).IndexOfAny("\\'\r\n\t"); if (idx >= 0) { _escapeIndices ??= new List(4); length += 4; _escapeIndices.Add(i + idx); var slice = value.Slice(i, idx).Span; length += encoding.GetByteCount(slice); i += idx + 1; } else { var slice = value.Slice(i).Span; length += encoding.GetByteCount(slice); i = value.Length; } } _value = value; _includeQuotes = includeQuotes; Length = length; } public int Write(Memory buffer) { Debug.Assert(buffer.Length >= Length); var encoding = Encoding.UTF8; int i = 0, bytesWritten = 0; if (_includeQuotes) { buffer.Span[bytesWritten++] = (byte)'\\'; buffer.Span[bytesWritten++] = (byte)'\''; } if (_escapeIndices != null) { foreach (var escapeIdx in _escapeIndices) { var slice = _value.Slice(i, escapeIdx - i); bytesWritten += encoding.GetBytes(slice.Span, buffer.Slice(bytesWritten).Span); var escapeBuffer = buffer.Slice(bytesWritten).Span; escapeBuffer[0] = (byte)'\\'; escapeBuffer[1] = (byte)'\\'; escapeBuffer[2] = (byte)'\\'; escapeBuffer[3] = (byte)(_value.Span[escapeIdx] switch { '\r' => 'r', '\n' => 'n', '\t' => 't', var c => c }); bytesWritten += 4; i = escapeIdx + 1; } } if (i < _value.Length) bytesWritten += encoding.GetBytes(_value.Slice(i).Span, buffer.Slice(bytesWritten).Span); if (_includeQuotes) { buffer.Span[bytesWritten++] = (byte)'\\'; buffer.Span[bytesWritten++] = (byte)'\''; } Debug.Assert(bytesWritten == Length); return bytesWritten; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringParameterWriter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class StringParameterWriter : IClickHouseParameterWriter> { private readonly IClickHouseColumnTypeInfo _type; public StringParameterWriter(IClickHouseColumnTypeInfo type) { _type = type; } public bool TryCreateParameterValueWriter(ReadOnlyMemory value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = new StringLiteralValueWriter(value, isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlyMemory value) { Interpolate(queryBuilder, value.Span); if (_type.ComplexTypeName != "String") queryBuilder.Append("::").Append(_type.ComplexTypeName); return queryBuilder; } public static StringBuilder Interpolate(StringBuilder queryBuilder, ReadOnlySpan stringSpan) { queryBuilder.Append('\''); foreach (var charValue in stringSpan) { switch (charValue) { case '\\': queryBuilder.Append("\\\\"); break; case '\'': queryBuilder.Append("''"); break; default: queryBuilder.Append(charValue); break; } } return queryBuilder.Append('\''); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _type, FunctionHelper.Apply); } public static StringParameterWriter Create(IClickHouseColumnTypeInfo typeInfo, string? format = null) where T : IFormattable { return new StringParameterWriter(typeInfo, value => value.ToString(format, CultureInfo.InvariantCulture).AsMemory()); } } internal sealed class StringParameterWriter : IClickHouseParameterWriter { private readonly IClickHouseColumnTypeInfo _type; private readonly Func> _toString; public StringParameterWriter(IClickHouseColumnTypeInfo type, Func> toString) { _type = type; _toString = toString; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = new StringLiteralValueWriter(_toString(value), isNested); return true; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { var str = _toString(value); StringParameterWriter.Interpolate(queryBuilder, str.Span); if (_type.ComplexTypeName != "String") queryBuilder.Append("::").Append(_type.ComplexTypeName); return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return writeValue(queryBuilder, _type, FunctionHelper.Apply); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Text; namespace Octonica.ClickHouseClient.Types { internal sealed class StringTableColumn : StringTableColumnBase { public override string DefaultValue => string.Empty; public StringTableColumn(Encoding encoding, List<(int segmentIndex, int offset, int length)> layouts, List> segments) : base(encoding, layouts, segments) { } protected override string GetValue(Encoding encoding, ReadOnlySpan span) { if (span.IsEmpty) return string.Empty; return encoding.GetString(span); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringTableColumnBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text; namespace Octonica.ClickHouseClient.Types { internal abstract class StringTableColumnBase : IClickHouseTableColumn, IClickHouseArrayTableColumn { private readonly Encoding _encoding; private readonly List<(int segmentIndex, int offset, int length)> _layouts; private readonly List> _segments; public int RowCount => _layouts.Count; public abstract TOut DefaultValue { get; } protected StringTableColumnBase(Encoding encoding, List<(int segmentIndex, int offset, int length)> layouts, List> segments) { _encoding = encoding ?? throw new ArgumentNullException(nameof(encoding)); _layouts = layouts ?? throw new ArgumentNullException(nameof(layouts)); _segments = segments ?? throw new ArgumentNullException(nameof(segments)); } public bool IsNull(int index) { return false; } [return: NotNull] public TOut GetValue(int index) { var(segmentIndex, offset, length) = _layouts[index]; if (length == 0) return GetValue(_encoding, Span.Empty); var span = _segments[segmentIndex].Slice(offset, length).Span; return GetValue(_encoding, span); } [return: NotNull] protected abstract TOut GetValue(Encoding encoding, ReadOnlySpan span); object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } public IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(byte[])) return (IClickHouseTableColumn)(object)new StringByteArrayTableColumn(_encoding, _layouts, _segments); if (typeof(T) == typeof(string)) return (IClickHouseTableColumn)(object)new StringTableColumn(_encoding, _layouts, _segments); if (typeof(T) == typeof(char[])) return (IClickHouseTableColumn)(object)new StringCharArrayTableColumn(_encoding, _layouts, _segments); return null; } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } public int CopyTo(int index, Span buffer, int dataOffset) { var (segmentIndex, offset, length) = _layouts[index]; if (dataOffset < 0 || dataOffset > length) throw new ArgumentOutOfRangeException(nameof(dataOffset)); if (length == 0) return 0; var maxLength = Math.Min(length - dataOffset, buffer.Length); var slice = _segments[segmentIndex].Slice(offset + dataOffset, maxLength); slice.Span.CopyTo(buffer); return maxLength; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StringTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class StringTypeInfo : SimpleTypeInfo { public StringTypeInfo() : base("String") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new StringColumnReader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new StringSkippingColumnReader(rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) == typeof(string)) return new StringColumnWriter(columnName, ComplexTypeName, (IReadOnlyList) rows, columnSettings?.StringEncoding ?? Encoding.UTF8); if (typeof(T) == typeof(char[])) { var mappedList = MappedReadOnlyList>.Map((IReadOnlyList)rows, m => m.AsMemory()); return new StringSpanColumnWriter(columnName, ComplexTypeName, mappedList, columnSettings?.StringEncoding ?? Encoding.UTF8); } if (typeof(T) == typeof(ReadOnlyMemory)) return new StringSpanColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>) rows, columnSettings?.StringEncoding ?? Encoding.UTF8); if (typeof(T) == typeof(Memory)) { var mappedList = MappedReadOnlyList, ReadOnlyMemory>.Map((IReadOnlyList>) rows, m => m); return new StringSpanColumnWriter(columnName, ComplexTypeName, mappedList, columnSettings?.StringEncoding ?? Encoding.UTF8); } if (typeof(T) == typeof(byte[])) return new BinaryStringColumnWriter(columnName, ComplexTypeName, (IReadOnlyList) rows); if (typeof(T) == typeof(ReadOnlyMemory)) return new BinaryStringSpanColumnWriter(columnName, ComplexTypeName, (IReadOnlyList>) rows); if (typeof(T) == typeof(Memory)) { var mappedList = MappedReadOnlyList, ReadOnlyMemory>.Map((IReadOnlyList>) rows, m => m); return new BinaryStringSpanColumnWriter(columnName, ComplexTypeName, mappedList); } throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(string)) writer = new StringParameterWriter(this, str => str.AsMemory()); else if (type == typeof(char[])) writer = new StringParameterWriter(this, arr => arr); else if (type == typeof(ReadOnlyMemory)) writer = new StringParameterWriter(this); else if (type == typeof(Memory)) writer = new StringParameterWriter>(this, mem => mem); else if (type == typeof(byte[])) writer = new HexStringParameterWriter(this, arr => arr); else if (type == typeof(ReadOnlyMemory)) writer = new HexStringParameterWriter(this); else if (type == typeof(Memory)) writer = new HexStringParameterWriter>(this, mem => mem); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(string); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.String; } private class StringColumnReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly int _bufferSize; private readonly List<(int segmentIndex, int offset, int length)> _layouts; private readonly List> _segments = new List>(1); private int _position; public StringColumnReader(int rowCount) { _rowCount = rowCount; _bufferSize = 4096; //TODO: from settings _layouts = new List<(int segmentIndex, int offset, int length)>(rowCount); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_layouts.Count >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); int expectedElementsCount = _rowCount - _layouts.Count; int elementsCount = 0, bytesCount = 0; while (elementsCount < expectedElementsCount) { var slice = sequence.Slice(bytesCount); if (!ClickHouseBinaryProtocolReader.TryRead7BitInteger(slice, out var longSize, out var bytesRead)) break; var size = (int) longSize; if (slice.Length - bytesRead < size) break; if (size == 0) { _layouts.Add((0, 0, 0)); } else { var lastSegment = _segments.Count == 0 ? default : _segments[^1]; lastSegment = lastSegment.Slice(_position); if (lastSegment.Length < size) { lastSegment = new Memory(new byte[Math.Max(_bufferSize, size)]); _position = 0; _segments.Add(lastSegment); } var stringBytes = slice.Slice(bytesRead, size); stringBytes.CopyTo(lastSegment.Span); _layouts.Add((_segments.Count - 1, _position, size)); _position += size; } ++elementsCount; bytesCount += size + bytesRead; } return new SequenceSize(bytesCount, elementsCount); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return new StringTableColumn(settings?.StringEncoding ?? Encoding.UTF8, _layouts, _segments); } } private sealed class StringSkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _rowCount; private int _position; public StringSkippingColumnReader(int rowCount) { _rowCount = rowCount; } public SequenceSize ReadNext(ReadOnlySequence sequence) { var maxElementsCount = _rowCount - _position; if (maxElementsCount <= 0) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); int offset = 0; int count = 0; while (count < maxElementsCount) { var slice = sequence.Slice(offset); if (!ClickHouseBinaryProtocolReader.TryRead7BitInteger(slice, out var size, out var bytesRead)) break; var totalLength = bytesRead + (int) size; if (slice.Length < totalLength) break; offset += totalLength; ++count; } _position += count; return new SequenceSize(offset, count); } } private sealed class BinaryStringColumnWriter : StringColumnWriterBase { private readonly IReadOnlyList _rows; protected override int RowCount => _rows.Count; public BinaryStringColumnWriter(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType) { _rows = rows; } protected override int GetByteCount(int rowIndex) { return _rows[rowIndex]?.Length ?? 0; } protected override void WriteBytes(int rowIndex, Span writeTo) { _rows[rowIndex].CopyTo(writeTo); } } private sealed class BinaryStringSpanColumnWriter : StringColumnWriterBase { private readonly IReadOnlyList> _rows; protected override int RowCount => _rows.Count; public BinaryStringSpanColumnWriter(string columnName, string columnType, IReadOnlyList> rows) : base(columnName, columnType) { _rows = rows; } protected override int GetByteCount(int rowIndex) { return _rows[rowIndex].Length; } protected override void WriteBytes(int rowIndex, Span writeTo) { _rows[rowIndex].Span.CopyTo(writeTo); } } private sealed class StringColumnWriter : StringColumnWriterBase { private readonly IReadOnlyList _rows; private readonly Encoding _encoding; protected override int RowCount => _rows.Count; public StringColumnWriter(string columnName, string columnType, IReadOnlyList rows, Encoding encoding) : base(columnName, columnType) { _rows = rows; _encoding = encoding; } protected override int GetByteCount(int rowIndex) { var str = _rows[rowIndex]; if (string.IsNullOrEmpty(str)) return 0; return _encoding.GetByteCount(str); } protected override void WriteBytes(int rowIndex, Span writeTo) { _encoding.GetBytes(_rows[rowIndex], writeTo); } } private sealed class StringSpanColumnWriter : StringColumnWriterBase { private readonly IReadOnlyList> _rows; private readonly Encoding _encoding; protected override int RowCount => _rows.Count; public StringSpanColumnWriter(string columnName, string columnType, IReadOnlyList> rows, Encoding encoding) : base(columnName, columnType) { _rows = rows; _encoding = encoding; } protected override int GetByteCount(int rowIndex) { return _encoding.GetByteCount(_rows[rowIndex].Span); } protected override void WriteBytes(int rowIndex, Span writeTo) { _encoding.GetBytes(_rows[rowIndex].Span, writeTo); } } private abstract class StringColumnWriterBase : IClickHouseColumnWriter { public string ColumnName { get; } public string ColumnType { get; } protected abstract int RowCount { get; } private int _position; protected StringColumnWriterBase(string columnName, string columnType) { ColumnName = columnName; ColumnType = columnType; } public SequenceSize WriteNext(Span writeTo) { var rowCount = RowCount; if (_position == rowCount) return new SequenceSize(0, 0); var result = new SequenceSize(0, 0); for (; _position < rowCount; _position++) { var span = writeTo.Slice(result.Bytes); if (span.IsEmpty) break; var byteCount = GetByteCount(_position); if (byteCount == 0) { span[0] = 0; result = new SequenceSize(result.Bytes + 1, result.Elements + 1); } else { var prefixLength = ClickHouseBinaryProtocolWriter.TryWrite7BitInteger(span, checked((ulong) byteCount)); if (prefixLength <= 0 || prefixLength + byteCount > span.Length) return result; WriteBytes(_position, span.Slice(prefixLength, byteCount)); result = new SequenceSize(result.Bytes + prefixLength + byteCount, result.Elements + 1); } } return result; } protected abstract int GetByteCount(int rowIndex); protected abstract void WriteBytes(int rowIndex, Span writeTo); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StructureReaderBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Diagnostics; using System.Runtime.InteropServices; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { /// /// Represents a base class capable of reading columns of value types. /// /// The type of the column that must be a value type (struct). /// The type of the output column. public abstract class StructureReaderBase : IClickHouseColumnReader where TIn : struct { private readonly int _rowCount; private int _position; private readonly TIn[]? _buffer; /// /// Gets the size of a single element in bytes. /// protected int ElementSize { get; } /// /// Gets the value indicating whether bytes from an input buffer can be copied to the column's buffer bitwise. The default is . /// protected virtual bool BitwiseCopyAllowed => false; /// /// Initializes with specified parameters. /// /// The size of a single element in bytes. /// The number of rows that the reader should read. protected StructureReaderBase(int elementSize, int rowCount) { ElementSize = elementSize; _rowCount = rowCount; if (rowCount > 0) _buffer = new TIn[rowCount]; } /// /// Reads as much elements as possible from the provided binary buffer. /// /// The binary buffer. /// The that contains the number of bytes and the number of elements which were read. public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var byteSize = Math.Min(ElementSize * (_rowCount - _position), (int) (sequence.Length - sequence.Length % ElementSize)); var elementCount = byteSize / ElementSize; if (elementCount == 0) return new SequenceSize(0, 0); int count; if (BitwiseCopyAllowed) { var targetBytes = MemoryMarshal.AsBytes(new Span(_buffer, _position, elementCount)); Debug.Assert(byteSize == targetBytes.Length); sequence.Slice(0, byteSize).CopyTo(targetBytes); count = elementCount; } else { count = CopyTo(sequence.Slice(0, byteSize), ((Span) _buffer).Slice(_position, elementCount)); Debug.Assert(count >= 0 && count <= elementCount); } _position += count; return new SequenceSize(count * ElementSize, count); } private int CopyTo(ReadOnlySequence source, Span target) { Span tmpSpan = stackalloc byte[ElementSize]; int count = 0; for (var slice = source; !slice.IsEmpty; slice = slice.Slice(ElementSize), count++) { if (slice.FirstSpan.Length >= ElementSize) target[count] = ReadElement(slice.FirstSpan); else { slice.Slice(0, ElementSize).CopyTo(tmpSpan); target[count] = ReadElement(tmpSpan); } } return count; } /// /// When overriden in a derived class reads a single element from the provided binary buffer. /// /// The binary buffer. /// The decoded value. protected abstract TIn ReadElement(ReadOnlySpan source); /// /// When overriden in a derived class creates a column for with the specified settings. /// /// The settings of the column. /// The buffer that contains the column's rows. /// A column for . protected abstract IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer); /// public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { return EndRead(settings, ((ReadOnlyMemory)_buffer).Slice(0, _position)); } IClickHouseTableColumn IClickHouseColumnReader.EndRead(ClickHouseColumnSettings? settings) { return EndRead(settings); } } /// /// Represents a base class capable of reading columns of value types. /// /// The type of the column that must be a value type (struct). public abstract class StructureReaderBase : StructureReaderBase where T : struct { /// /// Initializes with specified parameters. /// /// The size of a single element in bytes. /// The number of rows that the reader should read. public StructureReaderBase(int elementSize, int rowCount) : base(elementSize, rowCount) { } /// /// Creates a column for from the provided buffer. The column settings are ignored. /// /// The settings of the column. This argument is ignored by this method. /// The buffer that contains the column's rows. /// A column for . protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new StructureTableColumn(buffer); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StructureTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal class StructureTableColumn : IClickHouseTableColumn where T : struct { private readonly ReadOnlyMemory _buffer; public int RowCount => _buffer.Length; public T DefaultValue => default; public StructureTableColumn(ReadOnlyMemory buffer) { _buffer = buffer; } public bool IsNull(int index) { return false; } public T GetValue(int index) { return _buffer.Span[index]; } public virtual IClickHouseTableColumn? TryReinterpret() { if (typeof(TAs) == typeof(T?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, this); return null; } object IClickHouseTableColumn.GetValue(int index) { return GetValue(index); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out TRes dispatchedValue) { dispatchedValue = dispatcher.Dispatch(this); return true; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/StructureWriterBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { /// /// Represents a base class capable of writing columns of value types. /// /// The type of the column that must be a value type (struct). public abstract class StructureWriterBase : IClickHouseColumnWriter where T : struct { private readonly IReadOnlyList _rows; private int _position; /// /// Gets the size of a single element in bytes. /// protected int ElementSize { get; } /// public string ColumnName { get; } /// public string ColumnType { get; } /// /// Gets the value indicating whether elements from the column can be copied to an output buffer bitwise. The default is . /// protected virtual bool BitwiseCopyAllowed => false; /// /// Initializes with specified arguments. /// /// The name of the column to write data to. /// The full name of the ClickHouse type of the column. /// The size of a single element in bytes. /// The list of rows that the writer should write. protected StructureWriterBase(string columnName, string columnType, int elementSize, IReadOnlyList rows) { ElementSize = elementSize; _rows = rows; ColumnName = columnName; ColumnType = columnType; } /// public SequenceSize WriteNext(Span writeTo) { var elementsCount = Math.Min(_rows.Count - _position, writeTo.Length / ElementSize); if (BitwiseCopyAllowed) { var targetSpan = MemoryMarshal.Cast(writeTo.Slice(0, elementsCount * ElementSize)); var length = _rows.CopyTo(targetSpan, _position); Debug.Assert(length == elementsCount); _position += length; } else { for (int i = 0; i < elementsCount; i++, _position++) { WriteElement(writeTo.Slice(i * ElementSize), _rows[_position]); } } return new SequenceSize(elementsCount * ElementSize, elementsCount); } /// /// Writes a single value to the target span. /// /// The buffer to write a single value to. It's guaranteed that the size of the buffer is not less than the size of an element. /// The value that should be written. protected abstract void WriteElement(Span writeTo, in T value); } /// /// Represents a base class capable of converting column's values to a value type and writing them to a column. /// /// The type of the input data. /// The type of the column that must be a value type (struct). public abstract class StructureWriterBase : IClickHouseColumnWriter where TOut: struct { private readonly IReadOnlyList _rows; private int _position; /// /// Gets the size of a single element in bytes. /// protected int ElementSize { get; } /// public string ColumnName { get; } /// public string ColumnType { get; } /// /// Initializes with specified arguments. /// /// The name of the column to write data to. /// The full name of the ClickHouse type of the column. /// The size of a single element in bytes. /// The list of rows that the writer should write. protected StructureWriterBase(string columnName, string columnType, int elementSize, IReadOnlyList rows) { ElementSize = elementSize; _rows = rows; ColumnName = columnName; ColumnType = columnType; } /// public SequenceSize WriteNext(Span writeTo) { var elementsCount = Math.Min(_rows.Count - _position, writeTo.Length / ElementSize); var targetSpan = MemoryMarshal.Cast(writeTo.Slice(0, elementsCount * ElementSize)); _position += _rows.Map(Convert).CopyTo(targetSpan, _position); return new SequenceSize(elementsCount * ElementSize, elementsCount); } /// /// When overriden in a derived type converts a single element of the type /// to a value of the type . /// /// The value that should be converted. /// The value converted to the type . protected abstract TOut Convert(TIn value); } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/TupleTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal abstract class TupleTableColumnBase : IClickHouseTableColumn { public int RowCount { get; } protected TupleTableColumnBase(int rowCount) { RowCount = rowCount; } public abstract IEnumerable GetColumns(); protected abstract object GetTupleValue(int index); public bool IsNull(int index) { return false; } public object GetValue(int index) { return GetTupleValue(index); } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected void CheckIndex(int index) { if (index < 0 || index > RowCount) throw new ArgumentOutOfRangeException(nameof(index)); } public IClickHouseTableColumn? TryReinterpret() { var columns = GetColumns(); var columnList = columns as IReadOnlyList ?? columns.ToList(); return (IClickHouseTableColumn?) TryMakeTupleColumn(typeof(T), RowCount, columnList); } bool IClickHouseTableColumn.TryDipatch(IClickHouseTableColumnDispatcher dispatcher, out T dispatchedValue) { dispatchedValue = Dispatch(dispatcher); return true; } protected abstract T Dispatch(IClickHouseTableColumnDispatcher dispatcher); public static TupleTableColumnBase MakeTupleColumn(int rowCount, IReadOnlyList columns) { var columnElementTypes = new Type[columns.Count]; for (var i = 0; i < columns.Count; i++) columnElementTypes[i] = ClickHouseTableColumnHelper.TryGetValueType(columns[i]) ?? typeof(object); var tupleType = TupleTypeInfo.MakeTupleType(columnElementTypes); var tupleColumn = TryMakeTupleColumn(tupleType, rowCount, columns); if (tupleColumn == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. The column of the required type can't be created."); return tupleColumn; } private static TupleTableColumnBase? TryMakeTupleColumn(Type type, int rowCount, IReadOnlyList columns) { if (columns == null) throw new ArgumentNullException(nameof(columns)); if (!type.IsGenericType) return null; var typeDef = type.GetGenericTypeDefinition(); Type? reinterpreterTypeDef = null; switch (columns.Count) { case 1: if (typeDef == typeof(Tuple<>)) reinterpreterTypeDef = typeof(TupleTableColumn<>.Reinterpreter); else if (typeDef == typeof(ValueTuple<>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<>.Reinterpreter); break; case 2: if (typeDef == typeof(Tuple<,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,>.Reinterpreter); else if (typeDef == typeof(KeyValuePair<,>)) reinterpreterTypeDef = typeof(KeyValuePairTableColumn<,>.Reinterpreter); break; case 3: if (typeDef == typeof(Tuple<,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,>.Reinterpreter); break; case 4: if (typeDef == typeof(Tuple<,,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,,>.Reinterpreter); break; case 5: if (typeDef == typeof(Tuple<,,,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,,,>.Reinterpreter); break; case 6: if (typeDef == typeof(Tuple<,,,,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,,,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,,,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,,,,>.Reinterpreter); break; case 7: if (typeDef == typeof(Tuple<,,,,,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,,,,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,,,,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,,,,,>.Reinterpreter); break; default: if (columns.Count < 8) break; if (typeDef == typeof(Tuple<,,,,,,,>)) reinterpreterTypeDef = typeof(TupleTableColumn<,,,,,,,,>.Reinterpreter); else if (typeDef == typeof(ValueTuple<,,,,,,,>)) reinterpreterTypeDef = typeof(ValueTupleTableColumn<,,,,,,,,>.Reinterpreter); else break; var tuple8Args = type.GetGenericArguments(); var extraTupleType = tuple8Args[^1]; var extraColumn = TryMakeTupleColumn(extraTupleType, rowCount, columns.Slice(7)); if (extraColumn == null) return null; var extraColumnType = extraColumn.GetType(); if (!typeof(IClickHouseTableColumn<>).MakeGenericType(extraTupleType).IsAssignableFrom(extraColumnType)) return null; var tuple8ColumnTypeArgs = new Type[9]; tuple8Args.CopyTo(tuple8ColumnTypeArgs, 0); tuple8ColumnTypeArgs[^1] = extraColumnType; var tuple8ColumnInterpreter = (ReinterpreterBase) Activator.CreateInstance(reinterpreterTypeDef.MakeGenericType(tuple8ColumnTypeArgs))!; var tuple8Columns = new List(8); tuple8Columns.AddRange(columns.Take(7)); tuple8Columns.Add(extraColumn); return tuple8ColumnInterpreter.TryReinterpret(rowCount, tuple8Columns); } if (reinterpreterTypeDef == null) return null; var tupleArgs = type.GetGenericArguments(); var reinterpreterType = reinterpreterTypeDef.MakeGenericType(tupleArgs); var reinterpreter = (ReinterpreterBase) Activator.CreateInstance(reinterpreterType)!; return reinterpreter.TryReinterpret(rowCount, columns); } internal abstract class ReinterpreterBase { public abstract TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns); protected static IClickHouseTableColumn? TryReinterpret(IClickHouseTableColumn column) { var result = column as IClickHouseTableColumn ?? column.TryReinterpret(); if (result == null && typeof(T) == typeof(object)) return (IClickHouseTableColumn) (object) new ObjectColumnAdapter(column); return result; } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; public Tuple DefaultValue => Tuple.Create(_column1.DefaultValue); private TupleTableColumn(int rowCount, IClickHouseTableColumn column1) : base(rowCount) { _column1 = column1; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple(_column1.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 1); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; return new TupleTableColumn(rowCount, column1); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; public Tuple DefaultValue => Tuple.Create(_column1.DefaultValue, _column2.DefaultValue); private TupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2) : base(rowCount) { _column1 = column1; _column2 = column2; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple(_column1.GetValue(index), _column2.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 2); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; return new TupleTableColumn(rowCount, column1, column2); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; public Tuple DefaultValue => Tuple.Create(_column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue); private TupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 3); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; return new TupleTableColumn(rowCount, column1, column2, column3); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; public Tuple DefaultValue => Tuple.Create(_column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue); private TupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 4); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; return new TupleTableColumn(rowCount, column1, column2, column3, column4); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; public Tuple DefaultValue => Tuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue); private TupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 5); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; return new TupleTableColumn(rowCount, column1, column2, column3, column4, column5); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; public Tuple DefaultValue => Tuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue); private TupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 6); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; return new TupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6); } } } internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; private readonly IClickHouseTableColumn _column7; public Tuple DefaultValue => Tuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue, _column7.DefaultValue); private TupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6, IClickHouseTableColumn column7) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; _column7 = column7; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index), _column7.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; yield return _column7; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 7); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; var column7 = TryReinterpret(columns[6]); if (column7 == null) return null; return new TupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6, column7); } } } #pragma warning disable CS8714 // The type cannot be used as type parameter in the generic type or method. Nullability of type argument doesn't match 'notnull' constraint. internal sealed class TupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> where TColumnRest : TupleTableColumnBase, IClickHouseTableColumn { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; private readonly IClickHouseTableColumn _column7; private readonly TColumnRest _columnRest; public Tuple DefaultValue => new Tuple( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue, _column7.DefaultValue, _columnRest.DefaultValue); private TupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6, IClickHouseTableColumn column7, TColumnRest columnRest) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; _column7 = column7; _columnRest = columnRest; } public new Tuple GetValue(int index) { CheckIndex(index); return new Tuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index), _column7.GetValue(index), ((IClickHouseTableColumn) _columnRest).GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; yield return _column7; foreach (var extraColumn in _columnRest.GetColumns()) yield return extraColumn; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 8); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; var column7 = TryReinterpret(columns[6]); if (column7 == null) return null; var column8 = TryReinterpret(columns[7]); if (!(column8 is TColumnRest columnRest)) return null; return new TupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6, column7, columnRest); } } } #pragma warning restore CS8714 internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; public ValueTuple DefaultValue => ValueTuple.Create(_column1.DefaultValue); private ValueTupleTableColumn(int rowCount, IClickHouseTableColumn column1) : base(rowCount) { _column1 = column1; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple(_column1.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 1); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; return new ValueTupleTableColumn(rowCount, column1); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; public ValueTuple DefaultValue => ValueTuple.Create(_column1.DefaultValue, _column2.DefaultValue); private ValueTupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2) : base(rowCount) { _column1 = column1; _column2 = column2; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple(_column1.GetValue(index), _column2.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 2); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; public ValueTuple DefaultValue => ValueTuple.Create(_column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue); private ValueTupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 3); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; public ValueTuple DefaultValue => ValueTuple.Create(_column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue); private ValueTupleTableColumn(int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 4); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3, column4); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; public ValueTuple DefaultValue => ValueTuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue); private ValueTupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple(_column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 5); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3, column4, column5); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; public ValueTuple DefaultValue => ValueTuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue); private ValueTupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 6); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; private readonly IClickHouseTableColumn _column7; public ValueTuple DefaultValue => ValueTuple.Create( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue, _column7.DefaultValue); private ValueTupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6, IClickHouseTableColumn column7) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; _column7 = column7; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index), _column7.GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; yield return _column7; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 7); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; var column7 = TryReinterpret(columns[6]); if (column7 == null) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6, column7); } } } internal sealed class ValueTupleTableColumn : TupleTableColumnBase, IClickHouseTableColumn> where TRest : struct where TColumnRest : TupleTableColumnBase, IClickHouseTableColumn { private readonly IClickHouseTableColumn _column1; private readonly IClickHouseTableColumn _column2; private readonly IClickHouseTableColumn _column3; private readonly IClickHouseTableColumn _column4; private readonly IClickHouseTableColumn _column5; private readonly IClickHouseTableColumn _column6; private readonly IClickHouseTableColumn _column7; private readonly TColumnRest _columnRest; public ValueTuple DefaultValue => new ValueTuple( _column1.DefaultValue, _column2.DefaultValue, _column3.DefaultValue, _column4.DefaultValue, _column5.DefaultValue, _column6.DefaultValue, _column7.DefaultValue, _columnRest.DefaultValue); private ValueTupleTableColumn( int rowCount, IClickHouseTableColumn column1, IClickHouseTableColumn column2, IClickHouseTableColumn column3, IClickHouseTableColumn column4, IClickHouseTableColumn column5, IClickHouseTableColumn column6, IClickHouseTableColumn column7, TColumnRest columnRest) : base(rowCount) { _column1 = column1; _column2 = column2; _column3 = column3; _column4 = column4; _column5 = column5; _column6 = column6; _column7 = column7; _columnRest = columnRest; } public new ValueTuple GetValue(int index) { CheckIndex(index); return new ValueTuple( _column1.GetValue(index), _column2.GetValue(index), _column3.GetValue(index), _column4.GetValue(index), _column5.GetValue(index), _column6.GetValue(index), _column7.GetValue(index), ((IClickHouseTableColumn) _columnRest).GetValue(index)); } protected override object GetTupleValue(int index) { return GetValue(index); } public override IEnumerable GetColumns() { yield return _column1; yield return _column2; yield return _column3; yield return _column4; yield return _column5; yield return _column6; yield return _column7; foreach (var extraColumn in _columnRest.GetColumns()) yield return extraColumn; } protected override T Dispatch(IClickHouseTableColumnDispatcher dispatcher) { return dispatcher.Dispatch(this); } internal class Reinterpreter : ReinterpreterBase { public override TupleTableColumnBase? TryReinterpret(int rowCount, IReadOnlyList columns) { Debug.Assert(columns.Count == 8); var column1 = TryReinterpret(columns[0]); if (column1 == null) return null; var column2 = TryReinterpret(columns[1]); if (column2 == null) return null; var column3 = TryReinterpret(columns[2]); if (column3 == null) return null; var column4 = TryReinterpret(columns[3]); if (column4 == null) return null; var column5 = TryReinterpret(columns[4]); if (column5 == null) return null; var column6 = TryReinterpret(columns[5]); if (column6 == null) return null; var column7 = TryReinterpret(columns[6]); if (column7 == null) return null; var column8 = TryReinterpret(columns[7]); if (!(column8 is TColumnRest columnRest)) return null; return new ValueTupleTableColumn(rowCount, column1, column2, column3, column4, column5, column6, column7, columnRest); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/TupleTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class TupleTypeInfo : IClickHouseColumnTypeInfo { private readonly List? _elementTypes; private readonly List? _elementNames; public string ComplexTypeName { get; } public string TypeName => "Tuple"; public int GenericArgumentsCount => _elementTypes?.Count ?? 0; public TupleTypeInfo() { ComplexTypeName = TypeName; _elementTypes = null; } private TupleTypeInfo(string complexTypeName, List elementTypes, List? elementNames) { if (elementNames != null && elementTypes.Count != elementNames.Count) throw new ArgumentException("The number of elements must be equal to the number of element's types.", nameof(elementNames)); ComplexTypeName = complexTypeName; _elementTypes = elementTypes; _elementNames = elementNames; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var elementReaders = _elementTypes.Select(t => t.CreateColumnReader(rowCount)).ToList(); return new TupleColumnReader(rowCount, elementReaders); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateColumnReader(rowCount); if (serializationMode == ClickHouseColumnSerializationMode.Custom) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new TupleCustomSerializationColumnReader(this, rowCount); } throw new NotSupportedException($"The serialization mode {serializationMode} for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var elementReaders = _elementTypes.Select(t => t.CreateSkippingColumnReader(rowCount)).ToList(); return new TupleSkippingColumnReader(rowCount, elementReaders); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode == ClickHouseColumnSerializationMode.Default) return CreateSkippingColumnReader(rowCount); if (serializationMode == ClickHouseColumnSerializationMode.Custom) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new TupleCustomSerializationSkippingColumnReader(this, rowCount); } throw new NotSupportedException($"The serialization mode {serializationMode} for {TypeName} type is not supported by ClickHouseClient."); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return TupleColumnWriter.CreateColumnWriter(columnName, ComplexTypeName, _elementTypes, rows, columnSettings); } public IClickHouseParameterWriter CreateParameterWriter() { // TODO: ClickHouseDbType.Tuple is not supported in DefaultTypeInfoProvider.GetTypeInfo if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var tupeType = typeof(T); if (tupeType == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); return TupleColumnWriter.CreateParameterWriter(this, ComplexTypeName, _elementTypes, false); } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (_elementTypes != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); var complexTypeNameBuilder = new StringBuilder(TypeName).Append('('); var elementTypes = new List(options.Count); List? elementNames = null; foreach(var option in options) { if (elementTypes.Count > 0) complexTypeNameBuilder.Append(", "); var identifierLen = ClickHouseSyntaxHelper.GetIdentifierLiteralLength(option.Span); if (identifierLen == option.Span.Length) identifierLen = -1; if (identifierLen < 0) { if (elementNames != null) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, "A tuple can be either named or not. Mixing of named and unnamed arguments is not allowed."); var typeInfo = typeInfoProvider.GetTypeInfo(option); elementTypes.Add(typeInfo); } else { if (elementNames == null) { if (elementTypes.Count > 0) throw new ClickHouseException(ClickHouseErrorCodes.InvalidTypeName, "A tuple can be either named or not. Mixing of named and unnamed arguments is not allowed."); elementNames = new List(options.Count); } var name = ClickHouseSyntaxHelper.GetIdentifier(option.Span.Slice(0, identifierLen)); var typeInfo = typeInfoProvider.GetTypeInfo(option.Slice(identifierLen + 1)); elementTypes.Add(typeInfo); elementNames.Add(name); complexTypeNameBuilder.Append(option.Slice(0, identifierLen)).Append(' '); } complexTypeNameBuilder.Append(elementTypes[^1].ComplexTypeName); } var complexTypeName = complexTypeNameBuilder.Append(')').ToString(); return new TupleTypeInfo(complexTypeName, elementTypes, elementNames); } public Type GetFieldType() { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return MakeTupleType(_elementTypes.Select(elt => elt.GetFieldType()).ToArray()); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Tuple; } public IClickHouseTypeInfo GetGenericArgument(int index) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _elementTypes[index]; } public object GetTypeArgument(int index) { if (_elementTypes == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); if (_elementNames == null) return _elementTypes[index]; return new KeyValuePair(_elementNames[index], _elementTypes[index]); } public static Type MakeTupleType(IReadOnlyList genericArgs) { Type tupleType; switch (genericArgs.Count) { case 0: throw new ArgumentException("No type arguments specified.", nameof(genericArgs)); case 1: tupleType = typeof(Tuple<>); break; case 2: tupleType = typeof(Tuple<,>); break; case 3: tupleType = typeof(Tuple<,,>); break; case 4: tupleType = typeof(Tuple<,,,>); break; case 5: tupleType = typeof(Tuple<,,,,>); break; case 6: tupleType = typeof(Tuple<,,,,,>); break; case 7: tupleType = typeof(Tuple<,,,,,,>); break; default: tupleType = typeof(Tuple<,,,,,,,>); var restType = MakeTupleType(genericArgs.Slice(7)); var tuple8ArgsArr = new Type[8]; for (int i = 0; i < 7; i++) tuple8ArgsArr[i] = genericArgs[i]; tuple8ArgsArr[7] = restType; return tupleType.MakeGenericType(tuple8ArgsArr); } var genericArgsArr = genericArgs as Type[] ?? genericArgs.ToArray(); return tupleType.MakeGenericType(genericArgsArr); } private static void AddParameterWriter(List> writers, IClickHouseColumnTypeInfo elementType, Func getItem) { var elementWriter = elementType.CreateParameterWriter(); var writer = new TupleItemParameterWriter(elementWriter, getItem); writers.Add(writer); } private static IClickHouseParameterWriter CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList> itemWriters, bool isRest) { return new TupleParameterWriter(typeInfo, itemWriters, isRest); } private sealed class TupleCustomSerializationColumnReader : IClickHouseColumnReader { private readonly TupleTypeInfo _typeInfo; private readonly int _rowCount; private TupleColumnReader? _realReader; public TupleCustomSerializationColumnReader(TupleTypeInfo typeInfo, int rowCount) { _typeInfo = typeInfo; _rowCount = rowCount; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { if (_realReader != null) return ((IClickHouseColumnReaderBase)_realReader).ReadPrefix(sequence); var elementTypes = _typeInfo._elementTypes; Debug.Assert(elementTypes != null); if (sequence.Length < elementTypes.Count + 1) return SequenceSize.Empty; var mode = (ClickHouseColumnSerializationMode)sequence.FirstSpan[0]; if (mode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"The serialization mode {mode} is not supported by {_typeInfo.TypeName} type. Only the default mode is supported."); var seq = sequence.Slice(1); var typeModes = new List<(IClickHouseColumnTypeInfo type, ClickHouseColumnSerializationMode mode)>(elementTypes.Count); foreach (var type in elementTypes) { mode = (ClickHouseColumnSerializationMode)seq.FirstSpan[0]; if (mode != ClickHouseColumnSerializationMode.Default && mode != ClickHouseColumnSerializationMode.Sparse) throw new NotSupportedException($"Invalid serialization mode ({mode}) for an elment of tuple."); typeModes.Add((type, mode)); seq = seq.Slice(1); } var readers = typeModes.Select(tm => tm.type.CreateColumnReader(_rowCount, tm.mode)).ToList(); _realReader = new TupleColumnReader(_rowCount, readers); var result = ((IClickHouseColumnReaderBase)_realReader).ReadPrefix(seq); return result.AddBytes(elementTypes.Count + 1); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_realReader == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Detected an attempt to read the column before reading its prefix."); return _realReader.ReadNext(sequence); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { if (_realReader == null) return _typeInfo.CreateColumnReader(0).EndRead(settings); return _realReader.EndRead(settings); } } private sealed class TupleColumnReader : IClickHouseColumnReader { private readonly int _rowCount; private readonly IReadOnlyList _readers; private int _prefixPosition; private int _readerPosition; private int _position; public TupleColumnReader(int rowCount, IReadOnlyList readers) { _rowCount = rowCount; _readers = readers; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return ReadPrefix(sequence, _readers, ref _prefixPosition); } public SequenceSize ReadNext(ReadOnlySequence sequence) { var isLastColumn = _readerPosition == _readers.Count - 1; if (!isLastColumn && _position >= _rowCount) throw new ClickHouseException(ClickHouseErrorCodes.DataReaderError, "Internal error. Attempt to read after the end of the column."); var currentReader = _readers[_readerPosition]; var result = new SequenceSize(0, 0); while (!isLastColumn) { if (_position < _rowCount) { var elementsCount = _rowCount - _position; var size = currentReader.ReadNext(sequence.Slice(result.Bytes)); result = new SequenceSize(result.Bytes + size.Bytes, result.Elements); _position += size.Elements; if (size.Elements < elementsCount) return result; } _position = 0; currentReader = _readers[++_readerPosition]; isLastColumn = _readerPosition == _readers.Count - 1; } var lastColumnSize = currentReader.ReadNext(sequence.Slice(result.Bytes)); _position += lastColumnSize.Elements; return new SequenceSize(result.Bytes + lastColumnSize.Bytes, lastColumnSize.Elements); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { var columns = new List(_readers.Count); foreach(var reader in _readers) { var column = reader.EndRead(settings); columns.Add(column); } return TupleTableColumnBase.MakeTupleColumn(columns[^1].RowCount, columns); } public static SequenceSize ReadPrefix(ReadOnlySequence sequence, IReadOnlyList elementReaders, ref int prefixPosition) { var totalBytes = 0; var prefixReader = elementReaders[prefixPosition]; while (prefixPosition < elementReaders.Count - 1) { var prefixSize = prefixReader.ReadPrefix(sequence); totalBytes += prefixSize.Bytes; if (prefixSize.Elements == 0) return new SequenceSize(totalBytes, 0); if (prefixSize.Elements != 1) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Received an unexpected number of column prefixes: {prefixSize.Elements}."); prefixReader = elementReaders[++prefixPosition]; } var lastPrefixSize = prefixReader.ReadPrefix(sequence); return lastPrefixSize.AddBytes(totalBytes); } } private sealed class TupleCustomSerializationSkippingColumnReader : IClickHouseColumnReaderBase { private readonly TupleTypeInfo _typeInfo; private readonly int _rowCount; private TupleSkippingColumnReader? _realReader; public TupleCustomSerializationSkippingColumnReader(TupleTypeInfo typeInfo, int rowCount) { _typeInfo = typeInfo; _rowCount = rowCount; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { if (_realReader != null) return ((IClickHouseColumnReaderBase)_realReader).ReadPrefix(sequence); var elementTypes = _typeInfo._elementTypes; Debug.Assert(elementTypes != null); if (sequence.Length < elementTypes.Count + 1) return SequenceSize.Empty; var mode = (ClickHouseColumnSerializationMode)sequence.FirstSpan[0]; if (mode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"The serialization mode {mode} is not supported by {_typeInfo.TypeName} type. Only the default mode is supported."); var seq = sequence.Slice(1); var typeModes = new List<(IClickHouseColumnTypeInfo type, ClickHouseColumnSerializationMode mode)>(elementTypes.Count); foreach (var type in elementTypes) { mode = (ClickHouseColumnSerializationMode)seq.FirstSpan[0]; if (mode != ClickHouseColumnSerializationMode.Default && mode != ClickHouseColumnSerializationMode.Sparse) throw new NotSupportedException($"Invalid serialization mode ({mode}) for an elment of tuple."); typeModes.Add((type, mode)); seq = seq.Slice(1); } var readers = typeModes.Select(tm => tm.type.CreateSkippingColumnReader(_rowCount, tm.mode)).ToList(); _realReader = new TupleSkippingColumnReader(_rowCount, readers); var result = ((IClickHouseColumnReaderBase)_realReader).ReadPrefix(seq); return result.AddBytes(elementTypes.Count + 1); } public SequenceSize ReadNext(ReadOnlySequence sequence) { if (_realReader == null) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Detected an attempt to read the column before reading its prefix."); return _realReader.ReadNext(sequence); } } private sealed class TupleSkippingColumnReader : IClickHouseColumnReaderBase { private readonly int _rowCount; private readonly IReadOnlyList _elementReaders; private int _prefixPosition; private int _elementReaderIndex; private int _position; public TupleSkippingColumnReader(int rowCount, IReadOnlyList elementReaders) { _rowCount = rowCount; _elementReaders = elementReaders; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return TupleColumnReader.ReadPrefix(sequence, _elementReaders, ref _prefixPosition); } public SequenceSize ReadNext(ReadOnlySequence sequence) { var result = new SequenceSize(0, 0); var elementReader = _elementReaders[_elementReaderIndex]; while (_elementReaderIndex < _elementReaders.Count - 1) { var actualCount = elementReader.ReadNext(sequence.Slice(result.Bytes)); result = result.AddBytes(actualCount.Bytes); _position += actualCount.Elements; if (_position < _rowCount) return result; _position = 0; elementReader = _elementReaders[++_elementReaderIndex]; } var lastColumnCount = elementReader.ReadNext(sequence.Slice(result.Bytes)); _position += lastColumnCount.Elements; return lastColumnCount.AddBytes(result.Bytes); } } private sealed class TupleColumnWriter : IClickHouseColumnWriter { private readonly int _rowCount; private readonly List _columns; public string ColumnName { get; } public string ColumnType { get; } private int _prefixPosition; private int _currentWriterIdx; private int _currentWriterPosition; private TupleColumnWriter(string columnName, List columns, int rowCount) { _rowCount = rowCount; ColumnName = columnName; _columns = columns; var typeNameBuilder = _columns.Aggregate(new StringBuilder("Tuple("), (b, c) => b.Append(c.ColumnType).Append(',')); typeNameBuilder[^1] = ')'; ColumnType = typeNameBuilder.ToString(); } SequenceSize IClickHouseColumnWriter.WritePrefix(Span writeTo) { var totalBytes = 0; for (; _prefixPosition < _columns.Count; _prefixPosition++) { var slice = writeTo.Slice(totalBytes); var size = _columns[_prefixPosition].WritePrefix(slice); totalBytes += size.Bytes; if (size.Elements == 0) break; } if (_prefixPosition == _columns.Count) return new SequenceSize(totalBytes, 1); return new SequenceSize(totalBytes, 0); } public SequenceSize WriteNext(Span writeTo) { var result = new SequenceSize(0, 0); while (_currentWriterIdx < _columns.Count - 1) { var columnWriter = _columns[_currentWriterIdx]; if (_currentWriterPosition < _rowCount) { var expectedElementsCount = _rowCount - _currentWriterPosition; var actualCount = columnWriter.WriteNext(writeTo.Slice(result.Bytes)); _currentWriterPosition += actualCount.Elements; result = result.AddBytes(actualCount.Bytes); if (actualCount.Elements < expectedElementsCount) return result; } ++_currentWriterIdx; _currentWriterPosition = 0; } var lastColumnCount = _columns[^1].WriteNext(writeTo.Slice(result.Bytes)); _currentWriterPosition += lastColumnCount.Elements; return result.Add(lastColumnCount); } public static TupleColumnWriter CreateColumnWriter(string columnName, string columnType, IReadOnlyList elementTypes, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var factory = CreateWriterFactory(typeof(T), columnType, elementTypes); try { return factory.CreateColumnWriter(columnName, elementTypes, rows, columnSettings); } catch (Exception ex) { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{columnType}\".", ex); } } public static IClickHouseParameterWriter CreateParameterWriter(TupleTypeInfo typeInfo, string columnType, IReadOnlyList elementTypes, bool isRest) { var factory = CreateWriterFactory(typeof(T), columnType, elementTypes); object writer; try { writer = factory.CreateParameterWriter(typeInfo, elementTypes, isRest); } catch (Exception ex) { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{columnType}\".", ex); } return (IClickHouseParameterWriter)writer; } private static ITupleWriterFactory CreateWriterFactory(Type tupleType, string columnType, IReadOnlyList elementTypes) { Type? factoryType = null; if (tupleType.IsGenericType) { var listItemTypeDef = tupleType.GetGenericTypeDefinition(); switch (elementTypes.Count) { case 1: if (listItemTypeDef == typeof(Tuple<>)) factoryType = typeof(TupleColumnFactory<>); else if (listItemTypeDef == typeof(ValueTuple<>)) factoryType = typeof(ValueTupleColumnFactory<>); break; case 2: if (listItemTypeDef == typeof(Tuple<,>)) factoryType = typeof(TupleColumnFactory<,>); else if (listItemTypeDef == typeof(ValueTuple<,>)) factoryType = typeof(ValueTupleColumnFactory<,>); else if (listItemTypeDef == typeof(KeyValuePair<,>)) factoryType = typeof(KeyValuePairColumnFactory<,>); break; case 3: if (listItemTypeDef == typeof(Tuple<,,>)) factoryType = typeof(TupleColumnFactory<,,>); else if (listItemTypeDef == typeof(ValueTuple<,,>)) factoryType = typeof(ValueTupleColumnFactory<,,>); break; case 4: if (listItemTypeDef == typeof(Tuple<,,,>)) factoryType = typeof(TupleColumnFactory<,,,>); else if (listItemTypeDef == typeof(ValueTuple<,,,>)) factoryType = typeof(ValueTupleColumnFactory<,,,>); break; case 5: if (listItemTypeDef == typeof(Tuple<,,,,>)) factoryType = typeof(TupleColumnFactory<,,,,>); else if (listItemTypeDef == typeof(ValueTuple<,,,,>)) factoryType = typeof(ValueTupleColumnFactory<,,,,>); break; case 6: if (listItemTypeDef == typeof(Tuple<,,,,,>)) factoryType = typeof(TupleColumnFactory<,,,,,>); else if (listItemTypeDef == typeof(ValueTuple<,,,,,>)) factoryType = typeof(ValueTupleColumnFactory<,,,,,>); break; case 7: if (listItemTypeDef == typeof(Tuple<,,,,,,>)) factoryType = typeof(TupleColumnFactory<,,,,,,>); else if (listItemTypeDef == typeof(ValueTuple<,,,,,,>)) factoryType = typeof(ValueTupleColumnFactory<,,,,,,>); break; default: if (elementTypes.Count >= 8) { if (listItemTypeDef == typeof(Tuple<,,,,,,,>)) factoryType = typeof(TupleColumnFactory<,,,,,,,>); else if (listItemTypeDef == typeof(ValueTuple<,,,,,,,>)) factoryType = typeof(ValueTupleColumnFactory<,,,,,,,>); } break; } if (factoryType != null) { var args = tupleType.GetGenericArguments(); factoryType = factoryType.MakeGenericType(args); } } if (factoryType == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{tupleType}\" can't be converted to the ClickHouse type \"{columnType}\"."); ITupleWriterFactory? factory; try { factory = (ITupleWriterFactory?)Activator.CreateInstance(factoryType); } catch (Exception ex) { throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{tupleType}\" can't be converted to the ClickHouse type \"{columnType}\".", ex); } Debug.Assert(factory != null); return factory; } private interface ITupleWriterFactory { TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings); object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest); } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(1) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(2) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(3) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(4) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(5) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(6) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var columns = new List(7) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings), elementTypes[6].CreateColumnWriter(columnName, rows.Map(t => t.Item7), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); AddParameterWriter(elementWriters, elementTypes[6], t => t.Item7); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class TupleColumnFactory : ITupleWriterFactory where TRest : notnull { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var subColumns = elementTypes.Slice(7); var subType = "Tuple(" + string.Join(", ", subColumns.Select(c => c.ComplexTypeName)) + ")"; var lastColumn = TupleColumnWriter.CreateColumnWriter(columnName, subType, subColumns, rows.Map(t => t.Rest), columnSettings); var columns = new List(7 + lastColumn._columns.Count) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings), elementTypes[6].CreateColumnWriter(columnName, rows.Map(t => t.Item7), columnSettings) }; columns.AddRange(lastColumn._columns); return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); AddParameterWriter(elementWriters, elementTypes[6], t => t.Item7); var restElements = elementTypes.Slice(7); var restType = "Tuple(" + string.Join(", ", restElements.Select(c => c.ComplexTypeName)) + ")"; var restWriter = CreateParameterWriter(typeInfo, restType, restElements, true); elementWriters.Add(new TupleItemParameterWriter, TRest>(restWriter, t => t.Rest)); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(1) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(2) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(3) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(4) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(5) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(6) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(7) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings), elementTypes[6].CreateColumnWriter(columnName, rows.Map(t => t.Item7), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); AddParameterWriter(elementWriters, elementTypes[6], t => t.Item7); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class ValueTupleColumnFactory : ITupleWriterFactory where TRest : struct { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>) untypedRows; var subColumns = elementTypes.Slice(7); var subType = "Tuple(" + string.Join(", ", subColumns.Select(c => c.ComplexTypeName)) + ")"; var lastColumn = TupleColumnWriter.CreateColumnWriter(columnName, subType, subColumns, rows.Map(t => t.Rest), columnSettings); var columns = new List(7 + lastColumn._columns.Count) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(t => t.Item1), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(t => t.Item2), columnSettings), elementTypes[2].CreateColumnWriter(columnName, rows.Map(t => t.Item3), columnSettings), elementTypes[3].CreateColumnWriter(columnName, rows.Map(t => t.Item4), columnSettings), elementTypes[4].CreateColumnWriter(columnName, rows.Map(t => t.Item5), columnSettings), elementTypes[5].CreateColumnWriter(columnName, rows.Map(t => t.Item6), columnSettings), elementTypes[6].CreateColumnWriter(columnName, rows.Map(t => t.Item7), columnSettings) }; columns.AddRange(lastColumn._columns); return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var elementWriters = new List>>(elementTypes.Count); AddParameterWriter(elementWriters, elementTypes[0], t => t.Item1); AddParameterWriter(elementWriters, elementTypes[1], t => t.Item2); AddParameterWriter(elementWriters, elementTypes[2], t => t.Item3); AddParameterWriter(elementWriters, elementTypes[3], t => t.Item4); AddParameterWriter(elementWriters, elementTypes[4], t => t.Item5); AddParameterWriter(elementWriters, elementTypes[5], t => t.Item6); AddParameterWriter(elementWriters, elementTypes[6], t => t.Item7); var restElements = elementTypes.Slice(7); var restType = "Tuple(" + string.Join(", ", restElements.Select(c => c.ComplexTypeName)) + ")"; var restWriter = CreateParameterWriter(typeInfo, restType, restElements, true); elementWriters.Add(new TupleItemParameterWriter, TRest>(restWriter, t => t.Rest)); return TupleTypeInfo.CreateParameterWriter(typeInfo, elementWriters, isRest); } } private sealed class KeyValuePairColumnFactory : ITupleWriterFactory { public TupleColumnWriter CreateColumnWriter(string columnName, IReadOnlyList elementTypes, object untypedRows, ClickHouseColumnSettings? columnSettings) { var rows = (IReadOnlyList>)untypedRows; var columns = new List(2) { elementTypes[0].CreateColumnWriter(columnName, rows.Map(p => p.Key), columnSettings), elementTypes[1].CreateColumnWriter(columnName, rows.Map(p => p.Value), columnSettings) }; return new TupleColumnWriter(columnName, columns, rows.Count); } public object CreateParameterWriter(TupleTypeInfo typeInfo, IReadOnlyList elementTypes, bool isRest) { var writers = new List>>(2); AddParameterWriter(writers, elementTypes[0], pair => pair.Key); AddParameterWriter(writers, elementTypes[1], pair => pair.Value); return TupleTypeInfo.CreateParameterWriter(typeInfo, writers, isRest); } } } private sealed class TupleItemParameterWriter : IClickHouseParameterWriter { private readonly IClickHouseParameterWriter _itemWriter; private readonly Func _getItem; public TupleItemParameterWriter(IClickHouseParameterWriter itemWriter, Func getItem) { _itemWriter = itemWriter; _getItem = getItem; } public bool TryCreateParameterValueWriter(TTuple value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { return _itemWriter.TryCreateParameterValueWriter(_getItem(value), isNested, out valueWriter); } public StringBuilder Interpolate(StringBuilder queryBuilder, TTuple value) { return _itemWriter.Interpolate(queryBuilder, _getItem(value)); } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { return _itemWriter.Interpolate(queryBuilder, typeInfoProvider, writeValue); } } private sealed class TupleParameterWriter : IClickHouseParameterWriter { private readonly TupleTypeInfo _type; private readonly IReadOnlyList> _itemWriters; private readonly bool _isRest; public TupleParameterWriter(TupleTypeInfo type, IReadOnlyList> itemWriters, bool isRest) { _type = type; _itemWriters = itemWriters; _isRest = isRest; } public bool TryCreateParameterValueWriter(T value, bool isNested, [NotNullWhen(true)] out IClickHouseParameterValueWriter? valueWriter) { valueWriter = null; return false; } public StringBuilder Interpolate(StringBuilder queryBuilder, T value) { if(!_isRest) queryBuilder.Append("tuple("); bool isFirst = true; foreach(var itemWriter in _itemWriters) { if (isFirst) isFirst = false; else queryBuilder.Append(','); itemWriter.Interpolate(queryBuilder, value); } if (!_isRest) queryBuilder.Append(')'); return queryBuilder; } public StringBuilder Interpolate(StringBuilder queryBuilder, IClickHouseTypeInfoProvider typeInfoProvider, Func, StringBuilder>, StringBuilder> writeValue) { Debug.Assert(!_isRest); return writeValue(queryBuilder, _type, FunctionHelper.Apply); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt128TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal sealed class UInt128TypeInfo : BigIntegerTypeInfoBase { public UInt128TypeInfo() : base("UInt128", 128 / 8, true) { } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.UInt128; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt16TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt16TableColumn : StructureTableColumn { public UInt16TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(int)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(uint)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(ulong)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(int?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(uint?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(ulong?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt16TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt16TypeInfo : SimpleTypeInfo { public UInt16TypeInfo() : base("UInt16") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new UInt16Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(ushort), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList ushortRows; if (type == typeof(ushort)) ushortRows = (IReadOnlyList)rows; else if (type == typeof(byte)) ushortRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new UInt16Writer(columnName, ComplexTypeName, ushortRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(ushort)) writer = new SimpleParameterWriter(this, appendTypeCast: true); else if (type == typeof(byte)) writer = new SimpleParameterWriter(this, appendTypeCast: true, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(ushort); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.UInt16; } private sealed class UInt16Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public UInt16Reader(int rowCount) : base(sizeof(ushort), rowCount) { } protected override ushort ReadElement(ReadOnlySpan source) { return BitConverter.ToUInt16(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new UInt16TableColumn(buffer); } } private sealed class UInt16Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public UInt16Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(ushort), rows) { } protected override void WriteElement(Span writeTo, in ushort value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt256TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Types { internal sealed class UInt256TypeInfo : BigIntegerTypeInfoBase { public UInt256TypeInfo() : base("UInt256", 256 / 8, true) { } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.UInt256; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt32TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt32TableColumn : StructureTableColumn { public UInt32TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(ulong)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(ulong?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt32TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt32TypeInfo : SimpleTypeInfo { public UInt32TypeInfo() : base("UInt32") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new UInt32Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(uint), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList uintRows; if (type == typeof(uint)) uintRows = (IReadOnlyList)rows; else if (type == typeof(ushort)) uintRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) uintRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new UInt32Writer(columnName, ComplexTypeName, uintRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer = default(T) switch { uint _ => new SimpleParameterWriter(this, appendTypeCast: true), ushort _ => new SimpleParameterWriter(this, appendTypeCast: true, v => v), byte _ => new SimpleParameterWriter(this, appendTypeCast: true, v => v), _ => new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\".") }; return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(uint); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.UInt32; } private sealed class UInt32Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public UInt32Reader(int rowCount) : base(sizeof(uint), rowCount) { } protected override uint ReadElement(ReadOnlySpan source) { return BitConverter.ToUInt32(source); } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new UInt32TableColumn(buffer); } } private sealed class UInt32Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public UInt32Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(uint), rows) { } protected override void WriteElement(Span writeTo, in uint value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt64TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt64TypeInfo : SimpleTypeInfo { public UInt64TypeInfo() : base("UInt64") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new UInt64Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(ulong), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { var type = typeof(T); IReadOnlyList ulongRows; if (type == typeof(ulong)) ulongRows = (IReadOnlyList)rows; else if (type == typeof(uint)) ulongRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(ushort)) ulongRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else if (type == typeof(byte)) ulongRows = MappedReadOnlyList.Map((IReadOnlyList)rows, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new UInt64Writer(columnName, ComplexTypeName, ulongRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(ulong)) writer = new SimpleParameterWriter(this, appendTypeCast: true); else if (type == typeof(uint)) writer = new SimpleParameterWriter(this, appendTypeCast: true, v => v); else if (type == typeof(ushort)) writer = new SimpleParameterWriter(this, appendTypeCast: true, v => v); else if (type == typeof(byte)) writer = new SimpleParameterWriter(this, appendTypeCast: true, v => v); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(ulong); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.UInt64; } private sealed class UInt64Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public UInt64Reader(int rowCount) : base(sizeof(ulong), rowCount) { } protected override ulong ReadElement(ReadOnlySpan source) { return BitConverter.ToUInt64(source); } } internal sealed class UInt64Writer : StructureWriterBase { protected override bool BitwiseCopyAllowed => true; public UInt64Writer(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, sizeof(ulong), rows) { } protected override void WriteElement(Span writeTo, in ulong value) { var success = BitConverter.TryWriteBytes(writeTo, value); Debug.Assert(success); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt8TableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt8TableColumn : StructureTableColumn { public UInt8TableColumn(ReadOnlyMemory buffer) : base(buffer) { } public override IClickHouseTableColumn? TryReinterpret() { if (typeof(T) == typeof(short)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(ushort)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(int)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(uint)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(long)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(ulong)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v); if (typeof(T) == typeof(bool)) return (IClickHouseTableColumn) (object) new ReinterpretedTableColumn(this, v => v != 0); if (typeof(T) == typeof(short?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(ushort?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(int?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(uint?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(long?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(ulong?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v)); if (typeof(T) == typeof(bool?)) return (IClickHouseTableColumn) (object) new NullableStructTableColumn(null, new ReinterpretedTableColumn(this, v => v != 0)); return base.TryReinterpret(); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UInt8TypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; namespace Octonica.ClickHouseClient.Types { internal sealed class UInt8TypeInfo : SimpleTypeInfo { public UInt8TypeInfo() : base("UInt8") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new UInt8Reader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(sizeof(byte), rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { IReadOnlyList byteRows; if (typeof(T) == typeof(byte)) byteRows = (IReadOnlyList) rows; else if (typeof(T) == typeof(bool)) byteRows = MappedReadOnlyList.Map((IReadOnlyList) rows, v => v ? (byte) 1 : (byte) 0); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new UInt8Writer(columnName, ComplexTypeName, byteRows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); object writer; if (type == typeof(byte)) writer = new SimpleParameterWriter(this); else if (type == typeof(bool)) writer = new SimpleParameterWriter(this, b => b ? (byte)1 : (byte)0); else throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return (IClickHouseParameterWriter)writer; } public override Type GetFieldType() { return typeof(byte); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Byte; } private sealed class UInt8Reader : StructureReaderBase { protected override bool BitwiseCopyAllowed => true; public UInt8Reader(int rowCount) : base(sizeof(byte), rowCount) { } protected override byte ReadElement(ReadOnlySpan source) { return source[0]; } protected override IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings, ReadOnlyMemory buffer) { return new UInt8TableColumn(buffer); } } private sealed class UInt8Writer : IClickHouseColumnWriter { private readonly IReadOnlyList _rows; private int _position; public string ColumnName { get; } public string ColumnType { get; } public UInt8Writer(string columnName, string columnType, IReadOnlyList rows) { _rows = rows; ColumnName = columnName; ColumnType = columnType; } public SequenceSize WriteNext(Span writeTo) { var size = Math.Min(writeTo.Length, _rows.Count - _position); for (int i = 0; i < size; i++, _position++) writeTo[i] = _rows[_position]; return new SequenceSize(size, size); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/UuidTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Types { internal sealed class UuidTypeInfo : SimpleTypeInfo { private const int UuidSize = 16; public UuidTypeInfo() : base("UUID") { } public override IClickHouseColumnReader CreateColumnReader(int rowCount) { return new UuidReader(rowCount); } public override IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { return new SimpleSkippingColumnReader(UuidSize, rowCount); } public override IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (typeof(T) != typeof(Guid)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{typeof(T)}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); return new UuidWriter(columnName, ComplexTypeName, (IReadOnlyList)rows); } public override IClickHouseParameterWriter CreateParameterWriter() { var type = typeof(T); if (type == typeof(DBNull)) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The ClickHouse type \"{ComplexTypeName}\" does not allow null values."); if (type == typeof(Guid)) return (IClickHouseParameterWriter)(object)new StringParameterWriter(this, uuidValue => uuidValue.ToString().AsMemory()); throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, $"The type \"{type}\" can't be converted to the ClickHouse type \"{ComplexTypeName}\"."); } public override Type GetFieldType() { return typeof(Guid); } public override ClickHouseDbType GetDbType() { return ClickHouseDbType.Guid; } private sealed class UuidReader : StructureReaderBase { public UuidReader(int rowCount) : base(UuidSize, rowCount) { } protected override Guid ReadElement(ReadOnlySpan source) { ushort c = BitConverter.ToUInt16(source.Slice(0)); ushort b = BitConverter.ToUInt16(source.Slice(2)); uint a = BitConverter.ToUInt32(source.Slice(4)); return new Guid(a, b, c, source[15], source[14], source[13], source[12], source[11], source[10], source[9], source[8]); } } private sealed class UuidWriter:StructureWriterBase { public UuidWriter(string columnName, string columnType, IReadOnlyList rows) : base(columnName, columnType, UuidSize, rows) { } protected override void WriteElement(Span writeTo, in Guid value) { var success = value.TryWriteBytes(writeTo); Debug.Assert(success); var tmp = writeTo[0]; writeTo[0] = writeTo[6]; writeTo[6] = writeTo[2]; writeTo[2] = writeTo[4]; writeTo[4] = tmp; tmp = writeTo[1]; writeTo[1] = writeTo[7]; writeTo[7] = writeTo[3]; writeTo[3] = writeTo[5]; writeTo[5] = tmp; for (int i = 0; i < 4; i++) { tmp = writeTo[8 + i]; writeTo[8 + i] = writeTo[15 - i]; writeTo[15 - i] = tmp; } } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/VariantTableColumn.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Types { internal class VariantTableColumn : IClickHouseTableColumn { private readonly List _values; private readonly byte[] _columnIndices; private readonly int[] _valueIndices; public int RowCount => _columnIndices.Length; public VariantTableColumn(List values, byte[] columnIndices, int[] valueIndices) { Debug.Assert(columnIndices.Length == valueIndices.Length); _values = values; _columnIndices = columnIndices; _valueIndices = valueIndices; } public object GetValue(int index) { var columnIndex = _columnIndices[index]; // 0xFF stands for NULL if (columnIndex == 0xFF) return DBNull.Value; return _values[columnIndex].GetValue(_valueIndices[index]); } public bool IsNull(int index) { return _columnIndices[index] == 0xFF; } public bool TryDipatch(IClickHouseTableColumnDispatcher dispatcher, [MaybeNullWhen(false)] out T dispatchedValue) { dispatchedValue = default; return false; } public IClickHouseTableColumn? TryReinterpret() { return null; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Types/VariantTypeInfo.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Utils; using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; namespace Octonica.ClickHouseClient.Types { internal class VariantTypeInfo : IClickHouseColumnTypeInfo { private readonly List? _types; public string ComplexTypeName { get; } public string TypeName => "Variant"; public int GenericArgumentsCount => _types?.Count ?? 0; public VariantTypeInfo() { ComplexTypeName = TypeName; } private VariantTypeInfo(string complexTypeName, List types) { ComplexTypeName = complexTypeName; _types = types; } public IClickHouseColumnReader CreateColumnReader(int rowCount) { if (_types == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new VariantColumnReader(rowCount, this); } IClickHouseColumnReader IClickHouseColumnTypeInfo.CreateColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"Custom serialization for \"{ComplexTypeName}\" is not supported by ClickHouseClient."); return CreateColumnReader(rowCount); } public IClickHouseColumnReaderBase CreateSkippingColumnReader(int rowCount) { if (_types == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return new VariantSkippingColumnReader(rowCount, this); } IClickHouseColumnReaderBase IClickHouseColumnTypeInfo.CreateSkippingColumnReader(int rowCount, ClickHouseColumnSerializationMode serializationMode) { if (serializationMode != ClickHouseColumnSerializationMode.Default) throw new NotSupportedException($"Custom serialization for \"{ComplexTypeName}\" is not supported by ClickHouseClient."); return CreateSkippingColumnReader(rowCount); } public IClickHouseColumnWriter CreateColumnWriter(string columnName, IReadOnlyList rows, ClickHouseColumnSettings? columnSettings) { if (_types == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); var valueHandlers = new List(_types.Count); var nonGenericHandlerType = typeof(ValueHandler<>); foreach (var type in _types) { var clrType = type.GetFieldType(); var handler = (IValueHandler?)Activator.CreateInstance(nonGenericHandlerType.MakeGenericType(clrType)); Debug.Assert(handler != null); valueHandlers.Add(handler); } var indices = new byte[rows.Count]; int count = 0; foreach (var value in rows) { if (value == null || value is DBNull) { indices[count++] = 0xFF; continue; } bool handled = false; for (int i = 0; i < valueHandlers.Count; i++) { if (valueHandlers[i].TryHandle(value)) { indices[count++] = checked((byte)i); handled = true; break; } } if (!handled) { var allowedTypeList = string.Join(", ", _types.Select(t => $"\"{t.GetFieldType().Name}\"")); throw new ClickHouseException( ClickHouseErrorCodes.TypeNotSupported, $"Column \"{columnName}\". A value of type \"{value.GetType().Name}\" can't be written to the column of type \"{ComplexTypeName}\". Allowed types are: {allowedTypeList}."); } } Debug.Assert(count == indices.Length); var rowCounts = new List(valueHandlers.Count); var writers = new List(valueHandlers.Count); for (int i = 0; i < valueHandlers.Count; i++) { IValueHandler? handler = valueHandlers[i]; rowCounts.Add(handler.RowCount); writers.Add(handler.CreateColumnWriter(columnName, _types[i], columnSettings)); } return new VariantColumnWriter(columnName, ComplexTypeName, indices, rowCounts, writers); } public IClickHouseParameterWriter CreateParameterWriter() { throw new NotSupportedException($"Parameters of type \"{ComplexTypeName}\" are not supported."); } public ClickHouseDbType GetDbType() { return ClickHouseDbType.Variant; } public IClickHouseColumnTypeInfo GetDetailedTypeInfo(List> options, IClickHouseTypeInfoProvider typeInfoProvider) { if (_types != null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotSupported, "The type is already fully specified."); var types = new List(options.Count); var typeNameBuilder = new StringBuilder(TypeName).Append('('); bool isFirst = true; foreach(var option in options) { if (isFirst) isFirst = false; else typeNameBuilder.Append(", "); var type = typeInfoProvider.GetTypeInfo(option); types.Add(type); typeNameBuilder.Append(type.ComplexTypeName); } var complexTypeName = typeNameBuilder.Append(')').ToString(); return new VariantTypeInfo(complexTypeName, types); } public Type GetFieldType() { return typeof(object); } public IClickHouseTypeInfo GetGenericArgument(int index) { if (_types == null) throw new ClickHouseException(ClickHouseErrorCodes.TypeNotFullySpecified, $"The type \"{ComplexTypeName}\" is not fully specified."); return _types[index]; } private abstract class VariantColumnReaderBase where TReader:IClickHouseColumnReaderBase { private readonly List _prefixes; private IClickHouseColumnReaderBase? _prefixReader; private int _readerPosition; private int _rowPosition; protected readonly int RowCount; protected readonly VariantTypeInfo TypeInfo; protected readonly List Readers; public VariantColumnReaderBase(int rowCount, VariantTypeInfo typeInfo) { Debug.Assert(typeInfo._types != null); Readers = new List(typeInfo._types.Count); _prefixes = new List(typeInfo._types.Count); RowCount = rowCount; TypeInfo = typeInfo; } public SequenceSize ReadPrefix(ReadOnlySequence sequence) { var types = TypeInfo._types; Debug.Assert(types != null); var totalBytes = 0; for (int i = _prefixes.Count; i < types.Count; i++) { var slice = sequence.Slice(totalBytes); var reader = _prefixReader ?? types[i].CreateSkippingColumnReader(0); var prefixSize = reader.ReadPrefix(slice); if (prefixSize.Elements == 0) { _prefixReader = reader; return new SequenceSize(totalBytes, 0); } _prefixReader = null; totalBytes += prefixSize.Bytes; if (prefixSize.Bytes > 0) { var prefix = new byte[prefixSize.Bytes]; slice.Slice(0, prefix.Length).CopyTo(prefix); _prefixes.Add(prefix); } else { _prefixes.Add(Array.Empty()); } } return new SequenceSize(totalBytes, 1); } public SequenceSize ReadNext(ReadOnlySequence sequence, int[] rowCounts) { var seq = sequence; int totalBytes = 0; var totalElements = 0; Debug.Assert(TypeInfo._types != null); for (; _readerPosition < rowCounts.Length; _readerPosition++) { TReader reader; if (Readers.Count == _readerPosition) { reader = CreateReader(TypeInfo._types[_readerPosition], rowCounts[_readerPosition]); var prefixBytes = _prefixes[_readerPosition]; if (prefixBytes.Length > 0) { var prefixSize = reader.ReadPrefix(new ReadOnlySequence(prefixBytes)); if (prefixSize.Bytes == 0 && prefixSize.Elements == 0) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, "Internal error. Failed to read the column prefix."); if (prefixSize.Bytes != prefixBytes.Length) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. The column prefix' size is {prefixBytes.Length}, but the number of consumed bytes is {prefixSize.Bytes}."); if (prefixSize.Elements != 1) throw new ClickHouseException(ClickHouseErrorCodes.InternalError, $"Internal error. Received an unexpected number of column prefixes: {prefixSize.Elements}."); } Readers.Add(reader); } else { reader = Readers[^1]; } SequenceSize size; if (rowCounts[_readerPosition] == 0) size = SequenceSize.Empty; else size = reader.ReadNext(seq); totalBytes += size.Bytes; _rowPosition += size.Elements; if (_readerPosition == rowCounts.Length - 1) { totalElements += size.Elements; if (_rowPosition == rowCounts[_readerPosition]) totalElements += RowCount - rowCounts[_readerPosition]; } if (_rowPosition < rowCounts[_readerPosition]) break; _rowPosition = 0; seq = seq.Slice(size.Bytes); } return new SequenceSize(totalBytes, totalElements); } protected abstract TReader CreateReader(IClickHouseColumnTypeInfo typeInfo, int rowCount); } private sealed class VariantColumnReader : VariantColumnReaderBase, IClickHouseColumnReader { private readonly byte[] _typeIndices; private readonly int[] _elementIndices; private readonly int[] _rowCounts; private int _indexPosition; public VariantColumnReader(int rowCount, VariantTypeInfo typeInfo) :base(rowCount, typeInfo) { Debug.Assert(typeInfo._types != null); _typeIndices = rowCount == 0 ? Array.Empty() : new byte[rowCount]; _elementIndices = new int[rowCount]; _rowCounts = new int[typeInfo._types.Count]; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return ReadPrefix(sequence); } public SequenceSize ReadNext(ReadOnlySequence sequence) { int totalBytes = 0; var seq = sequence; if (_indexPosition < _typeIndices.Length) { var length = Math.Min(_typeIndices.Length - _indexPosition, (int)seq.Length); seq.Slice(0, length).CopyTo(((Span)_typeIndices).Slice(_indexPosition)); totalBytes += length; for (int i = 0; i < length; i++, _indexPosition++) { var typeIndex = _typeIndices[_indexPosition]; // 0xFF stands for NULL if (typeIndex != 0xFF) _elementIndices[_indexPosition] = _rowCounts[typeIndex]++; } if (_indexPosition < _typeIndices.Length) return new SequenceSize(totalBytes, 0); seq = seq.Slice(length); } var result = ReadNext(seq, _rowCounts); result = result.AddBytes(totalBytes); return result; } protected override IClickHouseColumnReader CreateReader(IClickHouseColumnTypeInfo typeInfo, int rowCount) { return typeInfo.CreateColumnReader(rowCount); } public IClickHouseTableColumn EndRead(ClickHouseColumnSettings? settings) { if (_typeIndices.Length == 0) return new StructureTableColumn(ReadOnlyMemory.Empty); Debug.Assert(TypeInfo._types != null); var columns = new List(_rowCounts.Length); for (int i = 0; i < TypeInfo._types.Count; i++) { if (i < Readers.Count) { columns.Add(Readers[i].EndRead(settings)); } else { var reader = TypeInfo._types[i].CreateColumnReader(0); columns.Add(reader.EndRead(settings)); } } return new VariantTableColumn(columns, _typeIndices, _elementIndices); } } private sealed class VariantSkippingColumnReader : VariantColumnReaderBase, IClickHouseColumnReaderBase { private readonly int[] _rowCounts; private int _indexPosition; public VariantSkippingColumnReader(int rowCount, VariantTypeInfo typeInfo) : base(rowCount, typeInfo) { Debug.Assert(typeInfo._types != null); _rowCounts = new int[typeInfo._types.Count]; } SequenceSize IClickHouseColumnReaderBase.ReadPrefix(ReadOnlySequence sequence) { return ReadPrefix(sequence); } public SequenceSize ReadNext(ReadOnlySequence sequence) { int totalBytes = 0; var seq = sequence; if (_indexPosition < RowCount) { var length = Math.Min(RowCount - _indexPosition, (int)seq.Length); var typeIndices = new byte[length]; seq.Slice(0, length).CopyTo(typeIndices); totalBytes += length; for (int i = 0; i < length; i++, _indexPosition++) { // 0xFF stands for NULL if (typeIndices[i] != 0xFF) ++_rowCounts[typeIndices[i]]; } if (_indexPosition < RowCount) return new SequenceSize(totalBytes, 0); seq = seq.Slice(length); } var result = ReadNext(seq, _rowCounts); result = result.AddBytes(totalBytes); return result; } protected override IClickHouseColumnReaderBase CreateReader(IClickHouseColumnTypeInfo typeInfo, int rowCount) { return typeInfo.CreateSkippingColumnReader(rowCount); } } private interface IValueHandler { int RowCount { get; } bool TryHandle(object value); IClickHouseColumnWriter CreateColumnWriter(string columnName, IClickHouseColumnTypeInfo typeInfo, ClickHouseColumnSettings? columnSettings); } private sealed class ValueHandler : IValueHandler { private readonly List _rows = new List(); public int RowCount => _rows.Count; public IClickHouseColumnWriter CreateColumnWriter(string columnName, IClickHouseColumnTypeInfo typeInfo, ClickHouseColumnSettings? columnSettings) { return typeInfo.CreateColumnWriter(columnName + "." + typeInfo.ComplexTypeName, _rows, columnSettings); } public bool TryHandle(object value) { if(value is T typedValue) { _rows.Add(typedValue); return true; } return false; } } private sealed class VariantColumnWriter : IClickHouseColumnWriter { private readonly byte[] _indices; private readonly List _rowCounts; private readonly List _columnWriters; private int _prefixPosition; private int _indexPosition; private int _columnPosition; private int _rowPosition; public string ColumnName { get; } public string ColumnType { get; } public VariantColumnWriter(string columnName, string columnType, byte[] indices, List rowCounts, List columnWriters) { ColumnName = columnName; ColumnType = columnType; _indices = indices; _rowCounts = rowCounts; _columnWriters = columnWriters; } SequenceSize IClickHouseColumnWriter.WritePrefix(Span writeTo) { var totalBytes = 0; for (; _prefixPosition < _columnWriters.Count; _prefixPosition++) { var prefixSize = _columnWriters[_prefixPosition].WritePrefix(writeTo.Slice(totalBytes)); totalBytes += prefixSize.Bytes; if (prefixSize.Elements == 0) return new SequenceSize(totalBytes, 0); } return new SequenceSize(totalBytes, 1); } public SequenceSize WriteNext(Span writeTo) { var totalBytes = 0; var span = writeTo; if (_indexPosition < _indices.Length) { var length = Math.Min(writeTo.Length, _indices.Length - _indexPosition); ((ReadOnlySpan)_indices).Slice(_indexPosition, length).CopyTo(span.Slice(0, length)); totalBytes += length; _indexPosition += length; if (_indexPosition < _indices.Length) return new SequenceSize(totalBytes, 0); span = span.Slice(length); } var totalElements = 0; for (; _columnPosition < _columnWriters.Count; _columnPosition++) { SequenceSize size; if (_rowCounts[_columnPosition] == 0) size = SequenceSize.Empty; else size = _columnWriters[_columnPosition].WriteNext(span); _rowPosition += size.Elements; totalBytes += size.Bytes; if (_columnPosition == _columnWriters.Count - 1) { totalElements += size.Elements; if (_rowPosition == _rowCounts[_columnPosition]) totalElements += _indices.Length - _rowPosition; } if (_rowPosition < _rowCounts[_columnPosition]) break; _rowPosition = 0; span = span.Slice(size.Bytes); } return new SequenceSize(totalBytes, totalElements); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/CertificateHelper.Net5.0.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET5_0_OR_GREATER using System.Runtime.CompilerServices; using System.Security.Cryptography.X509Certificates; namespace Octonica.ClickHouseClient.Utils { partial class CertificateHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] static partial void ImportPemCertificates(string filePath, X509Certificate2Collection collection) { collection.ImportFromPemFile(filePath); } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Utils/CertificateHelper.NetCoreApp3.1.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if !NET5_0_OR_GREATER using System.IO; using System.Security.Cryptography.X509Certificates; using System.Text; namespace Octonica.ClickHouseClient.Utils { partial class CertificateHelper { static partial void ImportPemCertificates(string filePath, X509Certificate2Collection collection) { /* PEM file without private keys looks like a bunch of concatenated CRT files * * -----BEGIN CERTIFICATE----- * Base64-encoded Certificate 1 * -----END CERTIFICATE----- * ... * -----BEGIN CERTIFICATE----- * Base64-encoded Certificate N * -----END CERTIFICATE----- */ using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); using var streamReader = new StreamReader(fs); var sb = new StringBuilder(); string? entity = null; int lineNumber = 0; const string delimiter = "-----"; const string beginPrefix = delimiter + "BEGIN "; const string endPrefix = delimiter + "END "; while (!streamReader.EndOfStream) { ++lineNumber; var line = streamReader.ReadLine(); if (string.IsNullOrWhiteSpace(line)) continue; if (entity == null) { if (!line.StartsWith(beginPrefix)) { var actualValue = line.Length <= beginPrefix.Length ? line : line.Substring(0, beginPrefix.Length); throw new InvalidDataException($"Unexpected value '{actualValue}' at the line {lineNumber}, position {0} of the certificate file. Expected value is '{beginPrefix}'."); } else if (!line.EndsWith(delimiter)) { var actualValue = line.Substring(line.Length - delimiter.Length); throw new InvalidDataException($"Unexpected value '{actualValue}' at the line {lineNumber}, position {line.Length - delimiter.Length} of the certificate file. Expected value is '{delimiter}'."); } entity = line.Substring(beginPrefix.Length, line.Length - beginPrefix.Length - delimiter.Length); if (string.IsNullOrWhiteSpace(entity)) { throw new InvalidDataException($"Missing non-empty value between '{beginPrefix}' and '{delimiter}' at the line {lineNumber}, position {beginPrefix.Length} of the certificate file."); } sb.AppendLine(line); } else if (!line.StartsWith(endPrefix)) { sb.AppendLine(line); } else if (!line.EndsWith(delimiter)) { var actualValue = line.Substring(line.Length - delimiter.Length); throw new InvalidDataException($"Unexpected value '{actualValue}' at the line {lineNumber}, position {line.Length - delimiter.Length} of the certificate file. Expected value is '{delimiter}'."); } else { var endEntity = line.Substring(endPrefix.Length, line.Length - endPrefix.Length - delimiter.Length); if (entity != endEntity) { throw new InvalidDataException($"The END value '{endEntity}' at the line {lineNumber}, position {beginPrefix.Length} of the certificate file is not equal to the BEGIN value '{entity}'."); } if (entity == "CERTIFICATE") { sb.AppendLine(line); collection.Import(Encoding.UTF8.GetBytes(sb.ToString())); } sb.Clear(); entity = null; } } if (entity != null) throw new InvalidDataException($"Unexpected end of the certificate file. Missing '{endPrefix}{entity}{delimiter}'."); } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Utils/CertificateHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.IO; using System.Security.Cryptography.X509Certificates; namespace Octonica.ClickHouseClient.Utils { internal static partial class CertificateHelper { public static X509Certificate2Collection LoadFromFile(string filePath) { var collection = new X509Certificate2Collection(); switch (Path.GetExtension(filePath).ToLowerInvariant()) { case ".pem": case ".crt": ImportPemCertificates(filePath, collection); break; default: collection.Import(filePath); break; } return collection; } static partial void ImportPemCertificates(string filePath, X509Certificate2Collection collection); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/CommonUtils.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.ObjectModel; using Octonica.ClickHouseClient.Protocol; namespace Octonica.ClickHouseClient.Utils { internal static class CommonUtils { internal static int GetColumnIndex(ReadOnlyCollection columns, string name) { int? caseInsensitiveIdx = null; for (int i = 0; i < columns.Count; i++) { if (string.Equals(columns[i].Name, name, StringComparison.Ordinal)) return i; if (string.Equals(columns[i].Name, name, StringComparison.OrdinalIgnoreCase)) { if (caseInsensitiveIdx == null) caseInsensitiveIdx = i; else caseInsensitiveIdx = -1; } } if (caseInsensitiveIdx >= 0) return caseInsensitiveIdx.Value; if (caseInsensitiveIdx == null) throw new IndexOutOfRangeException($"There is no column with the name \"{name}\" in the table."); throw new IndexOutOfRangeException($"There are two or more columns with the name \"{name}\" in the table. Please, provide an exact name (case-sensitive) of the column."); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ConstantReadOnlyList.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal class ConstantReadOnlyList : IReadOnlyListExt { [AllowNull] private readonly T _value; public int Count { get; } public ConstantReadOnlyList([AllowNull] T value, int count) { if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); Count = count; _value = value; } public IEnumerator GetEnumerator() { return Enumerable.Repeat(_value, Count).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (length < 0 || start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); return new ConstantReadOnlyList(_value, length); } public IReadOnlyListExt Map(Func map) { if (map == null) throw new ArgumentNullException(nameof(map)); return new ConstantReadOnlyList(map(_value), Count); } public int CopyTo(Span span, int offset) { if (offset < 0 || offset > Count) throw new ArgumentOutOfRangeException(nameof(offset)); var length = Math.Min(span.Length, Count - offset); span.Slice(0, length).Fill(_value); return length; } public T this[int index] { get { if (index >= 0 && index < Count) return _value; throw new IndexOutOfRangeException(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/FunctionHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Utils { internal static class FunctionHelper { public static TOut Apply(TIn input, Func func) => func(input); public static Func Combine(Func func1, Func func2) => v => func2(func1(v)); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ICollectionList.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; namespace Octonica.ClickHouseClient.Utils { internal interface ICollectionList { int Count { get; } T this[int listIndex, int index] { get; } IEnumerable GetItems(); IEnumerable GetListLengths(); int GetLength(int listIndex); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/IConverter.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Utils { internal interface IConverter { TOut Convert(TIn value); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/IConverterDispatcher.cs ================================================ #region License Apache 2.0 /* Copyright 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Utils { /// /// The interface for an object providing a type converter callback function. /// public interface IConverterDispatcher { /// /// When implemented, the method recieves the source type and an instance of an object for configuring with a callback function. /// /// The source type for a type converter callback function. /// The type of a configurable object. /// The dispatcher which recieves a type converter callback function and returns a configured object. /// The object configured with a type converter callback function. T Dispatch(IConverterDispatcher converterDispatcher); } /// /// The interface for an object receiving a type converter callback function. /// public interface IConverterDispatcher { /// /// The method configures an object with a type converter callback function. /// /// The source type of the type converter callback function. /// The result type of the type converter callback function. /// The type converter callback function /// The object configured with the type converter callback function. /// /// If and are the same type and the converter function is an identity function, call the method instead of this method. /// T Dispatch(Func convert); /// /// The method configures an object to not perform a type conversion. /// /// The object configured with no type converter. T DispatchNoConvert(); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/IReadOnlyListExt.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Utils { internal interface IReadOnlyListExt : IReadOnlyList { IReadOnlyListExt Slice(int start, int length); IReadOnlyListExt Map(Func map); int CopyTo(Span span, int start); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ITypeDispatcher.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Utils { internal interface ITypeDispatcher { T Dispatch(ITypeDispatcher dispatcher); } internal interface ITypeDispatcher { TOut Dispatch(); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/IndexedCollectionBase.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; namespace Octonica.ClickHouseClient.Utils { /// /// Represents a collection of items accessible both by key and index. /// /// The type of keys. /// The type of items. public abstract class IndexedCollectionBase : IList, IReadOnlyList, IReadOnlyDictionary, ICollection where TKey : notnull where TValue : notnull { private readonly Dictionary _items; private readonly List _keys; /// public int Count => _items.Count; bool ICollection.IsReadOnly => false; IEnumerable IReadOnlyDictionary.Values => this; IEnumerable IReadOnlyDictionary.Keys => _keys.AsReadOnly(); bool ICollection.IsSynchronized => false; object ICollection.SyncRoot => ((ICollection)_items).SyncRoot; /// /// Initializes a new instance of the class that is empty, has the /// default initial capacity, and uses the specified . /// /// /// The implementation to use when comparing keys, or to use the /// default for the type of the key. /// protected IndexedCollectionBase(IEqualityComparer? comparer) { _items = new Dictionary(comparer); _keys = new List(); } /// /// Initializes a new instance of the class that is empty, has the /// specified initial capacity, and uses the specified . /// /// The initial number of elements that the collection can contain. /// /// The implementation to use when comparing keys, or to use the /// default for the type of the key. /// protected IndexedCollectionBase(int capacity, IEqualityComparer? comparer) { _items = new Dictionary(capacity, comparer); _keys = new List(capacity); } /// /// Extracts and returns the key of the item. /// /// The item from which the key should be extracted. /// The key of the item. protected abstract TKey GetKey(TValue item); /// public bool ContainsKey(TKey key) { return _items.ContainsKey(key); } /// public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) { return _items.TryGetValue(key, out value); } /// /// Determines the index of the item with the specific key. /// /// The key of the item in the collection. /// The index of the item found in the collection; otherwise -1. public int IndexOf(TKey key) { if (key == null) throw new ArgumentNullException(nameof(key)); if (!_items.ContainsKey(key)) return -1; var comparer = _items.Comparer; return _keys.FindIndex(item => comparer.Equals(item, key)); } /// public int IndexOf(TValue item) { if (item == null) throw new ArgumentNullException(nameof(item)); var key = GetKey(item); if (!_items.TryGetValue(key, out var existingValue) || !existingValue.Equals(item)) return -1; var comparer = _items.Comparer; return _keys.FindIndex(item => comparer.Equals(item, key)); } /// public void Insert(int index, TValue item) { if (item == null) throw new ArgumentNullException(nameof(item)); var key = GetKey(item); _items.Add(key, item); try { _keys.Insert(index, key); } catch { _items.Remove(key); throw; } } /// public bool Remove(TValue item) { if (item == null) return false; var key = GetKey(item); if (!_items.TryGetValue(key, out var existingValue) || !existingValue.Equals(item)) return false; var comparer = _items.Comparer; var idx = _keys.FindIndex(item => comparer.Equals(item, key)); Debug.Assert(idx >= 0); RemoveAt(idx); return true; } /// /// Removes the item with the specific key from the collection. /// /// The key of the item in the collection. /// if the item was successfully removed from the collection. if an item was not found in the collection. public bool Remove(TKey key) { if (key == null) return false; if (!_items.TryGetValue(key, out var existingValue)) return false; var comparer = _items.Comparer; var idx = _keys.FindIndex(item => comparer.Equals(item, key)); Debug.Assert(idx >= 0); RemoveAt(idx); return true; } /// public void RemoveAt(int index) { var key = _keys[index]; _keys.RemoveAt(index); _items.Remove(key); } /// public void Add(TValue item) { if (item == null) throw new ArgumentNullException(nameof(item)); var key = GetKey(item); _items.Add(key, item); _keys.Add(key); } /// public void Clear() { _keys.Clear(); _items.Clear(); } /// public bool Contains(TValue item) { if (item == null) throw new ArgumentNullException(nameof(item)); var key = GetKey(item); if (!_items.TryGetValue(key, out var existingItem)) return false; return existingItem.Equals(item); } /// public void CopyTo(TValue[] array, int arrayIndex) { int sourceIdx = 0, targetIdx = arrayIndex; while (sourceIdx < _keys.Count) array[targetIdx++] = _items[_keys[sourceIdx++]]; } void ICollection.CopyTo(Array array, int index) { int sourceIdx = 0, targetIdx = index; while (sourceIdx < _keys.Count) array.SetValue(_items[_keys[sourceIdx++]], targetIdx++); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } IEnumerator> IEnumerable>.GetEnumerator() { foreach (var name in _keys) yield return new KeyValuePair(name, _items[name]); } /// public IEnumerator GetEnumerator() { foreach (var name in _keys) yield return _items[name]; } /// public TValue this[TKey key] => _items[key]; /// public TValue this[int index] { get => _items[_keys[index]]; set { if (value == null) throw new ArgumentNullException(nameof(value)); var valueKey = GetKey(value); var item = _items[_keys[index]]; var itemKey = GetKey(item); if (_items.Comparer.Equals(itemKey, valueKey)) { _items[valueKey] = value; Debug.Assert(_keys.Count == _items.Count); } else { _items.Add(valueKey, value); _items.Remove(itemKey); } _keys[index] = valueKey; } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ListExtensions.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Utils { internal static class ListExtensions { public static IReadOnlyList Map(this IReadOnlyList list, Func map) { return MappedReadOnlyList.Map(list, map); } public static IReadOnlyList Map(this IList list, Func map) { if (list == null) throw new ArgumentNullException(nameof(list)); return list.Slice(0).Map(map); } public static IReadOnlyList Slice(this IReadOnlyList list, int start) { if (list == null) throw new ArgumentNullException(nameof(list)); if (start == 0) return list; return list.Slice(start, list.Count - start); } public static IReadOnlyList Slice(this IReadOnlyList list, int start, int length) { if (list == null) throw new ArgumentNullException(nameof(list)); if (start == 0 && length == list.Count) return list; if (list is T[] array) return new ReadOnlyMemoryList(array).Slice(start, length); return ReadOnlyListSpan.Slice(list, start, length); } public static IReadOnlyList Slice(this IList list, int start) { if (list == null) throw new ArgumentNullException(nameof(list)); if (start == 0 && list is IReadOnlyList readOnlyList) return readOnlyList; return list.Slice(start, list.Count - start); } public static IReadOnlyList Slice(this IList list, int start, int length) { if (list == null) throw new ArgumentNullException(nameof(list)); if (start == 0 && list is IReadOnlyList readOnlyList && length == readOnlyList.Count) return readOnlyList; if (list is T[] array) return new ReadOnlyMemoryList(array).Slice(start, length); return ListSpan.Slice(list, start, length); } public static int CopyTo(this IReadOnlyList list, Span span, int start) { if (list == null) throw new ArgumentNullException(nameof(list)); if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.CopyTo(span, start); if (start < 0 || start > list.Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(list.Count - start, span.Length); if (list is T[] array) { new ReadOnlySpan(array, start, length).CopyTo(span); } else { var end = start + length; for (int i = start, j = 0; i < end; i++, j++) span[j] = list[i]; } return length; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ListSpan.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class ListSpan : IReadOnlyListExt { private readonly IList _innerList; private readonly int _offset; public int Count { get; } private ListSpan(IList innerList, int offset, int count) { if (innerList == null) throw new ArgumentNullException(nameof(innerList)); if (offset < 0 || offset > innerList.Count) throw new ArgumentOutOfRangeException(nameof(offset)); if (count < 0 || offset + count > innerList.Count) throw new ArgumentOutOfRangeException(nameof(count)); _innerList = innerList; _offset = offset; Count = count; } public IEnumerator GetEnumerator() { return _innerList.Skip(_offset).Take(Count).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); return new ListSpan(_innerList, start + _offset, length); } public IReadOnlyListExt Map(Func map) { return MappedListSpan.Create(_innerList, map, _offset, Count); } public int CopyTo(Span span, int start) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(Count - start, span.Length); if (_innerList is T[] array) { new ReadOnlySpan(array, _offset + start, length).CopyTo(span); } else { var end = _offset + start + length; for (int i = _offset + start, j = 0; i < end; i++, j++) span[j] = _innerList[i]; } return length; } public T this[int index] { get { if (index < 0 || index >= Count) throw new IndexOutOfRangeException(); return _innerList[index + _offset]; } } public static IReadOnlyListExt Slice(IList list, int offset, int count) { if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Slice(offset, count); return new ListSpan(list, offset, count); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/MappedListSpan.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class MappedListSpan : IReadOnlyListExt { private readonly IList _innerList; private readonly Func _map; private readonly int _offset; public int Count { get; } private MappedListSpan(IList innerList, Func map, int offset, int count) { if (innerList == null) throw new ArgumentNullException(nameof(innerList)); if (map == null) throw new ArgumentNullException(nameof(map)); if (offset < 0 || offset > innerList.Count) throw new ArgumentOutOfRangeException(nameof(offset)); if (count < 0 || offset + count > innerList.Count) throw new ArgumentOutOfRangeException(nameof(count)); _innerList = innerList; _map = map; _offset = offset; Count = count; } public IEnumerator GetEnumerator() { return _innerList.Skip(_offset).Take(Count).Select(_map).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (length < 0 || start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); return new MappedListSpan(_innerList, _map, _offset + start, length); } public IReadOnlyListExt Map(Func map) { if (map == null) throw new ArgumentNullException(nameof(map)); return new MappedListSpan(_innerList, Combine(_map, map), _offset, Count); static Func Combine(Func f1, Func f2) { return v => f2(f1(v)); } } public int CopyTo(Span span, int start) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(Count - start, span.Length); var end = _offset + start + length; for (int i = _offset + start, j = 0; i < end; i++, j++) span[j] = _map(_innerList[i]); return length; } public TOut this[int index] { get { if (index < 0 || index >= Count) throw new IndexOutOfRangeException(); return _map(_innerList[index + _offset]); } } public static IReadOnlyListExt Create(IList list, Func map, int offset, int count) { if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Slice(offset, count).Map(map); return new MappedListSpan(list, map, offset, count); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/MappedReadOnlyList.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class MappedReadOnlyList : IReadOnlyListExt { private readonly IReadOnlyList _innerList; private readonly Func _map; public int Count => _innerList.Count; private MappedReadOnlyList(IReadOnlyList innerList, Func map) { _innerList = innerList ?? throw new ArgumentNullException(nameof(innerList)); _map = map ?? throw new ArgumentNullException(nameof(map)); } public IEnumerator GetEnumerator() { return _innerList.Select(_map).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { return MappedReadOnlyListSpan.Create(_innerList, _map, start, length); } public IReadOnlyListExt Map(Func map) { if (map == null) throw new ArgumentNullException(nameof(map)); return new MappedReadOnlyList(_innerList, Combine(_map, map)); static Func Combine(Func f1, Func f2) { return v => f2(f1(v)); } } public int CopyTo(Span span, int start) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(Count - start, span.Length); var end = start + length; for (int i = start, j = 0; i < end; i++, j++) span[j] = _map(_innerList[i]); return length; } public TOut this[int index] => _map(_innerList[index]); public static IReadOnlyListExt Map(IReadOnlyList list, Func map) { if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Map(map); return new MappedReadOnlyList(list, map); } public static IReadOnlyListExt Map(ReadOnlyMemory memory, Func map) { return new MappedReadOnlyList(new ReadOnlyMemoryList(memory), map); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/MappedReadOnlyListSpan.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class MappedReadOnlyListSpan : IReadOnlyListExt { private readonly IReadOnlyList _innerList; private readonly Func _map; private readonly int _offset; public int Count { get; } private MappedReadOnlyListSpan(IReadOnlyList innerList, Func map, int offset, int count) { if (innerList == null) throw new ArgumentNullException(nameof(innerList)); if (map == null) throw new ArgumentNullException(nameof(map)); if (offset < 0 || offset > innerList.Count) throw new ArgumentOutOfRangeException(nameof(offset)); if (count < 0 || offset + count > innerList.Count) throw new ArgumentOutOfRangeException(nameof(count)); _innerList = innerList; _map = map; _offset = offset; Count = count; } public IEnumerator GetEnumerator() { return _innerList.Skip(_offset).Take(Count).Select(_map).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (length < 0 || start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); if (_innerList is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Slice(_offset + start, length).Map(_map); return new MappedReadOnlyListSpan(_innerList, _map, _offset + start, length); } public IReadOnlyListExt Map(Func map) { if (map == null) throw new ArgumentNullException(nameof(map)); return new MappedReadOnlyListSpan(_innerList, Combine(_map, map), _offset, Count); static Func Combine(Func f1, Func f2) { return v => f2(f1(v)); } } public int CopyTo(Span span, int start) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(Count - start, span.Length); var end = _offset + start + length; for (int i = _offset + start, j = 0; i < end; i++, j++) span[j] = _map(_innerList[i]); return length; } public TOut this[int index] { get { if (index < 0 || index >= Count) throw new IndexOutOfRangeException(); return _map(_innerList[index + _offset]); } } public static IReadOnlyListExt Create(IReadOnlyList list, Func map, int offset, int count) { if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Slice(offset, count).Map(map); if (list is TIn[] array) return new ReadOnlyMemoryList(array).Slice(offset, count).Map(map); return new MappedReadOnlyListSpan(list, map, offset, count); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/MemoryCollectionList.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; namespace Octonica.ClickHouseClient.Utils { internal sealed class MemoryCollectionList : ICollectionList { private readonly IReadOnlyList> _list; public MemoryCollectionList(IReadOnlyList> list) => _list = list; public int Count => _list.Count; public T this[int listIndex, int index] => _list[listIndex].Span[index]; public IEnumerable GetItems() => _list.SelectMany(list => MemoryMarshal.ToEnumerable(list)); public IEnumerable GetListLengths() => _list.Select(item => item.Length); public int GetLength(int listIndex) => _list[listIndex].Length; } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/MultiDimensionalArrayReadOnlyListAdapter.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Utils { internal static class MultiDimensionalArrayReadOnlyListAdapter { public static (Func createList, Type listElementType) Dispatch(Type arrayElementType, int arrayRank) { return TypeDispatcher.Dispatch(arrayElementType, new Dispatcher(arrayRank)); } private sealed class Dispatcher : ITypeDispatcher<(Func createList, Type listElementType)> { private readonly int _arrayRank; public Dispatcher(int arrayRank) { _arrayRank = arrayRank; } public (Func createList, Type listElementType) Dispatch() { return Dispatch( _arrayRank - 1, (arr, indices, idx) => { object? result; if (indices == null) { result = arr.GetValue(idx); } else { var idxCopy = new int[indices.Length + 1]; Array.Copy(indices, idxCopy, indices.Length); idxCopy[^1] = idx; result = arr.GetValue(idxCopy); } // Actually the result can be null if T is nullable return (T) result!; }); } private static (Func createList, Type listElementType) Dispatch(int depth, Func selector) { if (depth == 0) return (array => new Adapter(array, null, selector), typeof(T)); return Dispatch( depth - 1, (arr, indices, idx) => { int[] idxCopy; if (indices == null) { idxCopy = new[] {idx}; } else { idxCopy = new int[indices.Length + 1]; Array.Copy(indices, idxCopy, indices.Length); idxCopy[^1] = idx; } return new Adapter(arr, idxCopy, selector); }); } } private sealed class Adapter : IReadOnlyList { private readonly Array _array; private readonly int[]? _indices; private readonly Func _selector; public int Count => _array.GetLength(_indices?.Length ?? 0); public Adapter(Array array, int[]? indices, Func selector) { _array = array; _indices = indices; _selector = selector; } public T this[int index] => _selector(_array, _indices, index); public IEnumerator GetEnumerator() { var count = _array.GetLength(_indices?.Length ?? 0); for (int i = 0; i < count; i++) yield return _selector(_array, _indices, i); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ReadOnlyCollectionList.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class ReadOnlyCollectionList : ICollectionList { private readonly IReadOnlyList?> _list; public ReadOnlyCollectionList(IReadOnlyList?> list) => _list = list; public int Count => _list.Count; public T this[int listIndex, int index] => _list[listIndex]![index]; public IEnumerable GetItems() => _list.SelectMany(list => list ?? Enumerable.Empty()); public IEnumerable GetListLengths() => _list.Select(item => item?.Count ?? 0); public int GetLength(int listIndex) => _list[listIndex]?.Count ?? 0; } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ReadOnlyListSpan.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class ReadOnlyListSpan : IReadOnlyListExt { private readonly IReadOnlyList _innerList; private readonly int _offset; public int Count { get; } private ReadOnlyListSpan(IReadOnlyList innerList, int offset, int count) { if (innerList == null) throw new ArgumentNullException(nameof(innerList)); if (offset < 0 || offset > innerList.Count) throw new ArgumentOutOfRangeException(nameof(offset)); if (count < 0 || offset + count > innerList.Count) throw new ArgumentOutOfRangeException(nameof(count)); _innerList = innerList; _offset = offset; Count = count; } public IEnumerator GetEnumerator() { return _innerList.Skip(_offset).Take(Count).GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (length < 0 || start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); return Slice(_innerList, _offset + start, length); } public IReadOnlyListExt Map(Func map) { return MappedReadOnlyListSpan.Create(_innerList, map, _offset, Count); } public int CopyTo(Span span, int start) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(Count - start, span.Length); if (_innerList is T[] array) { new ReadOnlySpan(array, _offset + start, length).CopyTo(span); } else { var end = _offset + start + length; for (int i = _offset + start, j = 0; i < end; i++, j++) span[j] = _innerList[i]; } return length; } public T this[int index] { get { if (index < 0 || index >= Count) throw new IndexOutOfRangeException(); return _innerList[index + _offset]; } } public static IReadOnlyListExt Slice(IReadOnlyList list, int offset, int count) { if (list is IReadOnlyListExt readOnlyListExt) return readOnlyListExt.Slice(offset, count); return new ReadOnlyListSpan(list, offset, count); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ReadOnlyMemoryCollectionList.cs ================================================ #region License Apache 2.0 /* Copyright 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; namespace Octonica.ClickHouseClient.Utils { internal sealed class ReadOnlyMemoryCollectionList : ICollectionList { private readonly IReadOnlyList> _list; public ReadOnlyMemoryCollectionList(IReadOnlyList> list) => _list = list; public int Count => _list.Count; public T this[int listIndex, int index] => _list[listIndex].Span[index]; public IEnumerable GetItems() => _list.SelectMany(MemoryMarshal.ToEnumerable); public IEnumerable GetListLengths() => _list.Select(item => item.Length); public int GetLength(int listIndex) => _list[listIndex].Length; } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ReadOnlyMemoryList.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Utils { internal sealed class ReadOnlyMemoryList : IReadOnlyListExt { private readonly ReadOnlyMemory _memory; public int Count => _memory.Length; public ReadOnlyMemoryList(ReadOnlyMemory memory) { _memory = memory; } public ReadOnlyMemoryList(Memory memory) { _memory = memory; } public IEnumerator GetEnumerator() { for (int i = 0; i < _memory.Length; i++) yield return _memory.Span[i]; } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public IReadOnlyListExt Slice(int start, int length) { if (start < 0 || start > Count) throw new ArgumentOutOfRangeException(nameof(start)); if (length < 0 || start + length > Count) throw new ArgumentOutOfRangeException(nameof(length)); return new ReadOnlyMemoryList(_memory.Slice(start, length)); } public IReadOnlyListExt Map(Func map) { return MappedReadOnlyList.Map(_memory, map); } public int CopyTo(Span span, int start) { if (start < 0 || start > _memory.Length) throw new ArgumentOutOfRangeException(nameof(start)); var length = Math.Min(_memory.Length - start, span.Length); _memory.Slice(start, length).Span.CopyTo(span); return length; } public T this[int index] => _memory.Span[index]; } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/ReadWriteBuffer.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; namespace Octonica.ClickHouseClient.Utils { internal sealed class ReadWriteBuffer { private readonly int _segmentSize; private readonly List<(byte[] buffer, int length)> _segments; private int _readSegmentIdx; private int _readSegmentPosition; private int _readLength; private int _writeSegmentIdx; private int _writeSegmentPosition; private ReadOnlySequence? _readCache; public ReadWriteBuffer(int segmentSize) { if (segmentSize <= 0) throw new ArgumentOutOfRangeException(nameof(segmentSize)); _segmentSize = segmentSize; _segments = new List<(byte[] buffer, int length)>(); } public ReadOnlySequence Read() { if (_readCache != null) return _readCache.Value; if (_readLength == 0) return ReadOnlySequence.Empty; var readSegment = _segments[_readSegmentIdx]; if (readSegment.length - _readSegmentPosition >= _readLength) return new ReadOnlySequence(readSegment.buffer, _readSegmentPosition, _readLength); var memory = new List> {new ReadOnlyMemory(readSegment.buffer, _readSegmentPosition, readSegment.length - _readSegmentPosition)}; for (int length = _readLength - memory[0].Length, segmentIdx = (_readSegmentIdx + 1) % _segments.Count; length > 0; segmentIdx = (segmentIdx + 1) % _segments.Count) { var segment = _segments[segmentIdx]; var memoryBlock = new ReadOnlyMemory(segment.buffer, 0, Math.Min(segment.length, length)); memory.Add(memoryBlock); length -= memoryBlock.Length; } var sequenceSegment = new SimpleReadOnlySequenceSegment(memory); _readCache = new ReadOnlySequence(sequenceSegment, 0, sequenceSegment.LastSegment, sequenceSegment.LastSegment.Memory.Length); return _readCache.Value; } public void ConfirmRead(int length) { if (length < 0) throw new ArgumentOutOfRangeException(nameof(length)); if (length > _readLength) throw new ArgumentOutOfRangeException(nameof(length), "Internal error. The length can't be greater than the size of the buffer."); if (length == 0) return; var segment = _segments[_readSegmentIdx]; if (length < segment.length - _readSegmentPosition) { _readSegmentPosition += length; } else { var currentLength = length - segment.length + _readSegmentPosition; var currentIndex = _readSegmentIdx; var currentSegment = _segments[currentIndex]; while (true) { if (currentLength == 0 && currentIndex == _writeSegmentIdx && currentSegment.buffer.Length == _writeSegmentPosition) { // Read index is pointing to the beginning of the next segment. Write index is pointing to the end of // the current segment. Adjust write index. _writeSegmentIdx = _readSegmentIdx = (currentIndex + 1) % _segments.Count; _writeSegmentPosition = _readSegmentPosition = 0; currentSegment = _segments[_writeSegmentIdx]; _segments[_writeSegmentIdx] = (currentSegment.buffer, currentSegment.buffer.Length); break; } currentIndex = (currentIndex + 1) % _segments.Count; Debug.Assert(currentIndex != _readSegmentIdx); currentSegment = _segments[currentIndex]; if (currentLength < currentSegment.length) { _readSegmentIdx = currentIndex; _readSegmentPosition = currentLength; break; } currentLength -= currentSegment.length; } } _readLength -= length; if (_readCache != null) { _readCache = _readCache.Value.Slice(length); Debug.Assert(_readCache.Value.Length == _readLength); } } public Memory GetMemory() { if (_segments.Count > 0) { var writeSegment = _segments[_writeSegmentIdx]; if (writeSegment.buffer.Length > _writeSegmentPosition) return new Memory(writeSegment.buffer, _writeSegmentPosition, writeSegment.buffer.Length - _writeSegmentPosition); } return AcquireNextSegment(_segmentSize); } public Memory GetMemory(int sizeHint) { if (sizeHint < 0) throw new ArgumentOutOfRangeException(nameof(sizeHint)); if (_segments.Count > 0) { var writeSegment = _segments[_writeSegmentIdx]; if (writeSegment.buffer.Length - _writeSegmentPosition >= sizeHint) return new Memory(writeSegment.buffer, _writeSegmentPosition, writeSegment.buffer.Length - _writeSegmentPosition); } var segmentSize = _segmentSize; if (sizeHint > _segmentSize) segmentSize = (sizeHint / _segmentSize + Math.Min(sizeHint % _segmentSize, 1)) * _segmentSize; return AcquireNextSegment(segmentSize); } private Memory AcquireNextSegment(int size) { if (_segments.Count == 0) { _segments.Add((new byte[size], size)); return new Memory(_segments[_writeSegmentIdx].buffer); } _segments[_writeSegmentIdx] = (_segments[_writeSegmentIdx].buffer, _writeSegmentPosition); if (_segments.Count > 1) { var firstFreeSegmentIdx = (_writeSegmentIdx + 1) % _segments.Count; for (int i = firstFreeSegmentIdx; i != _readSegmentIdx; i = (i + 1) % _segments.Count) { if (_segments[i].buffer.Length >= size) { var segment = _segments[i]; if (i != firstFreeSegmentIdx) _segments[i] = _segments[firstFreeSegmentIdx]; _segments[firstFreeSegmentIdx] = (segment.buffer, segment.buffer.Length); _writeSegmentIdx = firstFreeSegmentIdx; _writeSegmentPosition = 0; return new Memory(_segments[_writeSegmentIdx].buffer); } } if (firstFreeSegmentIdx != _readSegmentIdx) { _segments[firstFreeSegmentIdx] = (new byte[size], size); _writeSegmentIdx = firstFreeSegmentIdx; _writeSegmentPosition = 0; return new Memory(_segments[_writeSegmentIdx].buffer); } } if (_writeSegmentIdx < _readSegmentIdx) ++_readSegmentIdx; ++_writeSegmentIdx; _writeSegmentPosition = 0; _segments.Insert(_writeSegmentIdx, (new byte[size], size)); return new Memory(_segments[_writeSegmentIdx].buffer); } public void ConfirmWrite(int length) { if (length < 0) throw new ArgumentOutOfRangeException(nameof(length)); var segment = _segments[_writeSegmentIdx]; if (length > segment.buffer.Length - _writeSegmentPosition) throw new ArgumentOutOfRangeException(nameof(length), "Internal error. The length can't be greater than the size of the buffer."); _writeSegmentPosition += length; Debug.Assert(_writeSegmentPosition <= segment.length); } public void Flush() { if (_segments.Count == 0) return; int length; if (_readSegmentIdx == _writeSegmentIdx) { length = _writeSegmentPosition - _readSegmentPosition; Debug.Assert(length >= 0); } else { length = _segments[_readSegmentIdx].length - _readSegmentPosition; for (int i = (_readSegmentIdx + 1) % _segments.Count; i != _writeSegmentIdx; i = (i + 1) % _segments.Count) length += _segments[i].length; length += _writeSegmentPosition; } if (length == _readLength) return; Debug.Assert(length > _readLength); _readCache = null; _readLength = length; } public void Discard() { if (_segments.Count == 0) return; int length = _readLength + _readSegmentPosition; int index = _readSegmentIdx; while (true) { var segment = _segments[index]; if (length <= segment.length) { _writeSegmentIdx = index; _writeSegmentPosition = length; _segments[index] = (segment.buffer, segment.buffer.Length); break; } length -= _segments[index].length; index = (index + 1) % _segments.Count; Debug.Assert(index != _readSegmentIdx); } } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/SimpleReadOnlySequenceSegment.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; namespace Octonica.ClickHouseClient.Utils { internal sealed class SimpleReadOnlySequenceSegment : ReadOnlySequenceSegment { private readonly SimpleReadOnlySequenceSegment? _lastSegment; public SimpleReadOnlySequenceSegment LastSegment => _lastSegment ?? this; public SimpleReadOnlySequenceSegment(ReadOnlyMemory firstSegment, ReadOnlySequence sequence) { var segments = new Stack>(2); long runningIndex = firstSegment.Length; if (firstSegment.Length > 0) segments.Push(firstSegment); foreach (var segment in sequence) { if (segment.Length == 0) continue; runningIndex += segment.Length; segments.Push(segment); } SimpleReadOnlySequenceSegment? nextSegment = null; if (segments.Count == 0) { Memory = firstSegment; Debug.Assert(runningIndex == 0); } else { while (segments.Count > 1) { var segment = segments.Pop(); runningIndex -= segment.Length; nextSegment = new SimpleReadOnlySequenceSegment(segment, runningIndex, nextSegment); } Memory = segments.Pop(); Debug.Assert(runningIndex == Memory.Length); } RunningIndex = 0; Next = nextSegment; _lastSegment = nextSegment?.LastSegment; } public SimpleReadOnlySequenceSegment(IReadOnlyList> segments) { var runningIndex = segments.Aggregate((long)0, (v, s) => v + s.Length); SimpleReadOnlySequenceSegment? nextSegment = null; for (int i = segments.Count - 1; i > 0; i--) { var segment = segments[i]; runningIndex -= segment.Length; nextSegment = new SimpleReadOnlySequenceSegment(segment, runningIndex, nextSegment); } if (segments.Count > 0) { var firstSegment = segments[0]; runningIndex -= firstSegment.Length; Memory = firstSegment; } Debug.Assert(runningIndex == 0); RunningIndex = runningIndex; Next = nextSegment; _lastSegment = nextSegment?.LastSegment; } private SimpleReadOnlySequenceSegment(ReadOnlyMemory memory, long runningIndex, SimpleReadOnlySequenceSegment? nextSegment) { Memory = memory; RunningIndex = runningIndex; Next = nextSegment; _lastSegment = nextSegment?.LastSegment; } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/TaskHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Diagnostics.CodeAnalysis; using System.Runtime.ExceptionServices; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Utils { internal static class TaskHelper { /// /// Call this method only for the task which is not async in any execution path /// internal static void WaitNonAsyncTask(ValueTask task) { try { task.AsTask().Wait(); } catch (AggregateException aggrEx) { if (aggrEx.InnerExceptions.Count == 1) ExceptionDispatchInfo.Capture(aggrEx.InnerExceptions[0]).Throw(); throw; } } /// /// Call this method only for the task which is not async in any execution path /// internal static T WaitNonAsyncTask(ValueTask task) { try { return task.Result; } catch (AggregateException aggrEx) { if (aggrEx.InnerExceptions.Count == 1) ExceptionDispatchInfo.Capture(aggrEx.InnerExceptions[0]).Throw(); throw; } } internal static T WaitSynchronously(Func> taskFactory) { if (taskFactory == null) throw new ArgumentNullException(nameof(taskFactory)); var innerTask = WaitInternal(taskFactory); try { var result = innerTask.Result; return result; } catch (AggregateException aggrEx) { if (aggrEx.InnerExceptions.Count == 1) ExceptionDispatchInfo.Capture(aggrEx.InnerExceptions[0]).Throw(); throw; } } internal static void WaitSynchronously([NotNull] Func taskFactory) { if (taskFactory == null) throw new ArgumentNullException(nameof(taskFactory)); var innerTask = WaitInternal(taskFactory); try { innerTask.Wait(); } catch (AggregateException aggrEx) { if (aggrEx.InnerExceptions.Count == 1) ExceptionDispatchInfo.Capture(aggrEx.InnerExceptions[0]).Throw(); throw; } } private static async Task WaitInternal([NotNull] Func> taskFactory) { return await Task.Run(async () => await taskFactory().ConfigureAwait(false)).ConfigureAwait(false); } private static async Task WaitInternal([NotNull] Func taskFactory) { await Task.Run(async () => await taskFactory().ConfigureAwait(false)).ConfigureAwait(false); } } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/TimeZoneHelper.Net6.0.cs ================================================ #region License Apache 2.0 /* Copyright 2021-2022 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NET6_0_OR_GREATER using System; using System.Runtime.CompilerServices; namespace Octonica.ClickHouseClient.Utils { partial class TimeZoneHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] static partial void GetTimeZoneInfoImpl(string timeZone, ref TimeZoneInfo? timeZoneInfo) { timeZoneInfo = TimeZoneInfo.FindSystemTimeZoneById(timeZone); } [MethodImpl(MethodImplOptions.AggressiveInlining)] static partial void GetTimeZoneIdImpl(TimeZoneInfo timeZoneInfo, ref string? timeZoneCode) { if (timeZoneInfo.HasIanaId) { timeZoneCode = timeZoneInfo.Id; } else if (!TimeZoneInfo.TryConvertWindowsIdToIanaId(timeZoneInfo.Id, out timeZoneCode)) { throw new TimeZoneNotFoundException($"The IANA time zone identifier for the time zone '{timeZoneInfo.Id}' was not found."); } } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Utils/TimeZoneHelper.NetCoreApp3.1.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion #if NETCOREAPP3_1_OR_GREATER && !NET6_0_OR_GREATER using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using TimeZoneConverter; namespace Octonica.ClickHouseClient.Utils { partial class TimeZoneHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] static partial void GetTimeZoneInfoImpl(string timeZone, ref TimeZoneInfo? timeZoneInfo) { timeZoneInfo = TZConvert.GetTimeZoneInfo(timeZone); } [MethodImpl(MethodImplOptions.AggressiveInlining)] static partial void GetTimeZoneIdImpl(TimeZoneInfo timeZoneInfo, ref string? timeZoneCode) { timeZoneCode = timeZoneInfo.Id; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) timeZoneCode = TZConvert.WindowsToIana(timeZoneCode); } } } #endif ================================================ FILE: src/Octonica.ClickHouseClient/Utils/TimeZoneHelper.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Utils { internal static partial class TimeZoneHelper { public static TimeZoneInfo GetTimeZoneInfo(string timeZone) { TimeZoneInfo? timeZoneInfo = null; GetTimeZoneInfoImpl(timeZone, ref timeZoneInfo); if (timeZoneInfo == null) throw new NotImplementedException($"Internal error. The method {nameof(GetTimeZoneInfoImpl)} is not implemented properly."); return timeZoneInfo; } public static string GetTimeZoneId(TimeZoneInfo timeZone) { string? timeZoneCode = null; GetTimeZoneIdImpl(timeZone, ref timeZoneCode); if (timeZoneCode == null) throw new NotImplementedException($"Internal error. The method {nameof(GetTimeZoneIdImpl)} is not implemented properly."); return timeZoneCode; } static partial void GetTimeZoneInfoImpl(string timeZone, ref TimeZoneInfo? timeZoneInfo); static partial void GetTimeZoneIdImpl(TimeZoneInfo timeZoneInfo, ref string? timeZoneCode); } } ================================================ FILE: src/Octonica.ClickHouseClient/Utils/TypeDispatcher.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Utils { internal static class TypeDispatcher { public static TOut Dispatch(Type type, ITypeDispatcher dispatcher) { var dispatcherType = typeof(Dispatcher<>).MakeGenericType(type); var dsp = (ITypeDispatcher) Activator.CreateInstance(dispatcherType)!; return dsp.Dispatch(dispatcher); } public static ITypeDispatcher Create(Type type) { var dispatcherType = typeof(Dispatcher<>).MakeGenericType(type); var dsp = (ITypeDispatcher)Activator.CreateInstance(dispatcherType)!; return dsp; } private class Dispatcher : ITypeDispatcher { public T Dispatch(ITypeDispatcher dispatcher) { return dispatcher.Dispatch(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Benchmarks/ColumnWriterBenchmarks.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; namespace Octonica.ClickHouseClient.Benchmarks { public class ColumnWriterBenchmarks { private ClickHouseConnectionSettings _connectionSettings; private List _id; private List _str; private List _dt; private List _val; [Params(10_000, 50_000)] public int Rows; [Params(1, 100, 1000)] public int BatchSize; [GlobalSetup] public void Setup() { _connectionSettings = ConnectionSettingsHelper.GetConnectionSettings(); using (var connection = new ClickHouseConnection(_connectionSettings)) { connection.Open(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ColumnWriterInsertBenchmarks"); cmd.ExecuteNonQuery(); cmd.CommandText = "CREATE TABLE ColumnWriterInsertBenchmarks(id UUID, str Nullable(String), dt DateTime, val Decimal64(4)) ENGINE = Memory"; cmd.ExecuteNonQuery(); } _id = new List(Rows); _str = new List(Rows); _dt = new List(Rows); _val = new List(Rows); const string strSource = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; var now = DateTime.Now; var rnd = new Random(2306); for (int i = 0; i < Rows; i++) { _id.Add(Guid.NewGuid()); _str.Add(i % strSource.Length == 0 ? null : strSource.Substring(0, i % strSource.Length)); _dt.Add(now.AddSeconds(i)); _val.Add(Math.Round((decimal) (rnd.NextDouble() - 0.5) * 100_000, 4)); } } [Benchmark] public async Task WriteBatсhes() { await using var connection = new ClickHouseConnection(_connectionSettings); await connection.OpenAsync(); await using var writer = connection.CreateColumnWriter("INSERT INTO ColumnWriterInsertBenchmarks VALUES"); var id = new List(BatchSize); var str = new List(BatchSize); var dt = new List(BatchSize); var val = new List(BatchSize); var columns = new object[4]; columns[writer.GetOrdinal("id")] = id; columns[writer.GetOrdinal("str")] = str; columns[writer.GetOrdinal("dt")] = dt; columns[writer.GetOrdinal("val")] = val; for (int i = 0; i < Rows; i += BatchSize) { id.Clear(); str.Clear(); dt.Clear(); val.Clear(); var len = Math.Min(BatchSize, Rows - i); for (int j = 0; j < len; j++) { id.Add(_id[i + j]); str.Add(_str[i + j]); dt.Add(_dt[i + j]); val.Add(_val[i + j]); } await writer.WriteTableAsync(columns, len, CancellationToken.None); } await writer.EndWriteAsync(CancellationToken.None); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Benchmarks/InsertRowBenchmark.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; namespace Octonica.ClickHouseClient.Benchmarks { public class InsertRowBenchmark { private ClickHouseConnectionSettings _connectionSettings; private List _id; private List _str; private List _dt; private List _val; [Params(50, 200)] public int Rows; [Params(true, false)] public bool KeepConnection; [GlobalSetup] public void Setup() { _connectionSettings = ConnectionSettingsHelper.GetConnectionSettings(); using (var connection = new ClickHouseConnection(_connectionSettings)) { connection.Open(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ColumnWriterInsertBenchmarks"); cmd.ExecuteNonQuery(); cmd.CommandText = "CREATE TABLE ColumnWriterInsertBenchmarks(id UUID, str Nullable(String), dt DateTime, val Decimal64(4)) ENGINE = Memory"; cmd.ExecuteNonQuery(); } _id = new List(Rows); _str = new List(Rows); _dt = new List(Rows); _val = new List(Rows); const string strSource = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; var now = DateTime.Now; var rnd = new Random(2306); for (int i = 0; i < Rows; i++) { _id.Add(Guid.NewGuid()); _str.Add(i % strSource.Length == 0 ? null : strSource.Substring(0, i % strSource.Length)); _dt.Add(now.AddSeconds(i)); _val.Add(Math.Round((decimal)(rnd.NextDouble() - 0.5) * 100_000, 4)); } } [Benchmark] public async Task InsertWithColumnWriter() { var id = new List(1) {Guid.Empty}; var str = new List(1) {null}; var dt = new List(1) {DateTime.MinValue}; var val = new List(1) {0}; if (KeepConnection) { await using var connection = new ClickHouseConnection(_connectionSettings); await connection.OpenAsync(); for (int i = 0; i < Rows; i++) { await using var writer = connection.CreateColumnWriter("INSERT INTO ColumnWriterInsertBenchmarks VALUES"); var columns = new object[4]; columns[writer.GetOrdinal("id")] = id; columns[writer.GetOrdinal("str")] = str; columns[writer.GetOrdinal("dt")] = dt; columns[writer.GetOrdinal("val")] = val; id[0] = _id[i]; str[0] = _str[i]; dt[0] = _dt[i]; val[0] = _val[i]; await writer.WriteTableAsync(columns, 1, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } } else { for (int i = 0; i < Rows; i++) { await using var connection = new ClickHouseConnection(_connectionSettings); await connection.OpenAsync(); await using var writer = connection.CreateColumnWriter("INSERT INTO ColumnWriterInsertBenchmarks VALUES"); var columns = new object[4]; columns[writer.GetOrdinal("id")] = id; columns[writer.GetOrdinal("str")] = str; columns[writer.GetOrdinal("dt")] = dt; columns[writer.GetOrdinal("val")] = val; id[0] = _id[i]; str[0] = _str[i]; dt[0] = _dt[i]; val[0] = _val[i]; await writer.WriteTableAsync(columns, 1, CancellationToken.None); //await writer.EndWriteAsync(CancellationToken.None); } } } [Benchmark] public async Task InsertFromSelect() { if (KeepConnection) { await using var connection = new ClickHouseConnection(_connectionSettings); await connection.OpenAsync(); for (int i = 0; i < Rows; i++) { await using var cmd = connection.CreateCommand("INSERT INTO ColumnWriterInsertBenchmarks(id, str, dt, val) SELECT {id:UUID}, {str}, {dt}, {val}"); var idParam = new ClickHouseParameter("id") {Value = _id[i].ToString(), ClickHouseDbType = ClickHouseDbType.String}; cmd.Parameters.Add(idParam); var strParam = new ClickHouseParameter("str") {Value = _str[i], ClickHouseDbType = ClickHouseDbType.String}; cmd.Parameters.Add(strParam); var dtParam = new ClickHouseParameter("dt") {Value = _dt[i], ClickHouseDbType = ClickHouseDbType.DateTime}; cmd.Parameters.Add(dtParam); var valParam = new ClickHouseParameter("val") {Value = _val[i], ClickHouseDbType = ClickHouseDbType.Decimal}; cmd.Parameters.Add(valParam); await cmd.ExecuteNonQueryAsync(); } } else { for (int i = 0; i < Rows; i++) { await using var connection = new ClickHouseConnection(_connectionSettings); await connection.OpenAsync(); await using var cmd = connection.CreateCommand("INSERT INTO ColumnWriterInsertBenchmarks(id, str, dt, val) SELECT {id:UUID}, {str}, {dt}, {val}"); var idParam = new ClickHouseParameter("id") {Value = _id[i].ToString(), ClickHouseDbType = ClickHouseDbType.String}; cmd.Parameters.Add(idParam); var strParam = new ClickHouseParameter("str") {Value = _str[i], ClickHouseDbType = ClickHouseDbType.String}; cmd.Parameters.Add(strParam); var dtParam = new ClickHouseParameter("dt") {Value = _dt[i], ClickHouseDbType = ClickHouseDbType.DateTime}; cmd.Parameters.Add(dtParam); var valParam = new ClickHouseParameter("val") {Value = _val[i], ClickHouseDbType = ClickHouseDbType.Decimal}; cmd.Parameters.Add(valParam); await cmd.ExecuteNonQueryAsync(); if (!KeepConnection) { await cmd.DisposeAsync(); await connection.DisposeAsync(); } } } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Benchmarks/Octonica.ClickHouseClient.Benchmarks.csproj ================================================  netcoreapp3.1;net6.0;net8.0 Exe enable Always ================================================ FILE: src/Octonica.ClickHouseClient.Benchmarks/Program.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using BenchmarkDotNet.Running; namespace Octonica.ClickHouseClient.Benchmarks { internal static class Program { private static void Main() { BenchmarkRunner.Run(typeof(Program).Assembly); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/AsyncTestFibSequence.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Tests { internal sealed class AsyncTestFibSequence : IAsyncEnumerable { public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = new CancellationToken()) { return new Enumerator(); } private sealed class Enumerator : IAsyncEnumerator { private decimal _prev; private decimal _next = 1; public decimal Current { get; private set; } public ValueTask DisposeAsync() { return default; } public ValueTask MoveNextAsync() { _prev = Current; Current = _next; _next = _prev + Current; return new ValueTask(true); } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/CityHashTests.Data.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; namespace Octonica.ClickHouseClient.Tests { partial class CityHashTests { // Expected values were genereated with the reference implementation of CityHash // https://github.com/ClickHouse/ClickHouse/blob/master/contrib/cityhash102/include/city.h private static readonly UInt64[,] testdata = new UInt64[kTestSize, 16] { { C(0x9ae16a3b2f90404f), C(0x75106db890237a4a), C(0x3feac5f636039766), C(0x3df09dfc64c09a2b), C(0x3cb540c392e51e29), C(0x6b56343feac0663), C(0x5b7bc50fd8e8ad92), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa4c09bd9f0e45f0), C(0xeb83988ef66c657c), C(0x6b9ccf8681a18aa1), C(0x535daa5e388d3a90), C(0x74836eeafb7f7102), C(0x2a57492128885367), C(0xebc8ac93ca0466d2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3be5e76f9931c681), C(0xb06fdde338ed51e1), C(0x47b3ccffbf2f6d07), C(0x5e54cfca59ba7e38), C(0xea2779909528f985), C(0x6c161777d6a8023), C(0xd1f3c5fb9996bc00), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1e9cc7d232c7a33f), C(0x236de88d608d66b4), C(0x43352ab6411f608), C(0x6315f182053dd353), C(0x9c8d9f1e3e37158a), C(0x1136a36ed4a9ffd9), C(0x393f1316596cf0be), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6f1d652a9aa9382e), C(0xd5abb3825f0f754), C(0x7f51135ccd215f9a), C(0x6425ead2a90c7a2b), C(0x44037551d679aee), C(0xce5d08d7367b62d1), C(0x4795915c38e590f8), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3bd60c14f66bb809), C(0xb16e895fd5f365ec), C(0x51075d0889c013d2), C(0x6ad28d3665a43295), C(0x7fc37ed943e66dc8), C(0xad299ebe643bfb48), C(0x5bd04fbf925f6cce), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbcbf47b8a67165c2), C(0xe0675209accbb9be), C(0x7308de500106588b), C(0x352b4f3f0772b9c7), C(0x6792aebe7398c194), C(0x95204503ef64f633), C(0xb6e4a03d70c094fc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe067868dc58fd396), C(0xc8473c58086a01ed), C(0x360edd83935faa24), C(0x93c93a1af22dc970), C(0x809dd5b41b66c5e6), C(0x9f1d9d910cf52399), C(0xdbe481a61d1e7afe), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6479c218e3b4a905), C(0xb70e630314f69053), C(0x91fbca514877b918), C(0x14049ebb94229b31), C(0xc3077ff5341e2606), C(0x40225f5626645bc0), C(0x7f1e41d06b6617bf), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x18009929c10980cf), C(0x1228aeb4388c73ce), C(0x161bf0c40170a6cf), C(0x62b03ff682658646), C(0xe6980864fd5d231c), C(0x522896654f825861), C(0xe0e758f820a5469c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xddf30a6fdce5d271), C(0xef58ff5ce35daf9a), C(0xd7e9b5802f25800e), C(0xb499363261c302d), C(0x101d05196bfd326b), C(0x2b3bcb83b73cbe96), C(0x962ea2a5be845d42), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc949210e09de308), C(0xa3a3a00d5e0bb375), C(0xd35d45563d3f80ee), C(0xf1f641038e0144a), C(0x5683a2db245de92a), C(0xab52a459db446624), C(0x6b20fd105557de34), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xdf87f848054247cb), C(0x161ecfb8370fd8d8), C(0x370313fa6788665e), C(0xeb71b5269045213b), C(0x62fb1a8ff15dfce5), C(0x150a9394f3fdb96b), C(0xcc0d5d64a6e18f2c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6a0cb058805e4acd), C(0x301106bab8fc482d), C(0x4cf37c5a1e5798cb), C(0xd356b4644394283d), C(0x571aae72bac35532), C(0xc850f195952feda6), C(0x78414943a6ae6f0c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9433ace8412f255f), C(0x25c9902d7dcfcf86), C(0x4c10ac88117aa0a8), C(0x8babd16f6370ed49), C(0x7a7c7fa1019e2ebb), C(0x67a5714b7b594c9b), C(0x22d631afd621c2fd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x723768cf3fa7b3a8), C(0x268a28711d788ba1), C(0xfb3792ee19b2da32), C(0x63bbd31777fbc65d), C(0x1b313de6f5b010ab), C(0xcb3434ed701a4f15), C(0x54594ab884cc93ae), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd3362226822b1b09), C(0x54f213c110832024), C(0x58b8afceea142bea), C(0x38a6adc421343bda), C(0xdeac2b566cc7b6c7), C(0xbd9562ad101d7afa), C(0xa5609ca846fd88e1), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1dc2612576924bbb), C(0x5cf79870310a596a), C(0x8f1cc739a9efcc7), C(0x818c48dc83dd96df), C(0x46a1ff36301aa443), C(0x23bbde43de2cb214), C(0xa8c333112a243c8c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5ee23a15102f6d57), C(0x6504de0d98e50ad5), C(0x5be10f78492f612a), C(0x298a7ba24ea61876), C(0xce1a0c9da8aa0d75), C(0x6b5a9a8f64ee1da6), C(0x9f74e86c6da69421), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9744fe3e5efa3407), C(0xc57f2bce8c2900cf), C(0x3456d73508cd04d7), C(0x5079f7dfed35a42d), C(0x141901f72878884f), C(0x491e400491cd4ece), C(0x7c19d3530ea3547f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe91e4a718994579c), C(0x31ae7ead8f1e2a04), C(0x343568a4ee4195be), C(0xd466bd089dc336b7), C(0x90eb9e1272e261a6), C(0x5f8b04a15ae42361), C(0xfc193363336453dd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x32fd044b0766c3db), C(0xcd99c8567fafb46), C(0xb690bcc63bd8e56f), C(0xc22e4983680d6d0a), C(0xfc9a888ff205bf15), C(0xeb6ecbb0b831d185), C(0xe0168df5fad0c670), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb410fced2be12203), C(0x81741151bc486ee0), C(0x345169c236f0b0fa), C(0x9c9e3686134f0592), C(0x6ceeb3c6dbebee95), C(0x4e7f425bfac67ca7), C(0x9461b911a1c6d589), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa6a3e76de568fb61), C(0x1e350f999671b7da), C(0xb16703528b8ac03d), C(0x8c414acd75ca5463), C(0xa65ea81aeb84b69), C(0x210c7500995aa0e6), C(0x6c13190557106457), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd169c59771e4a833), C(0xb5611e98efa87d2c), C(0x6be952171b8dccd1), C(0xb64af6857280d273), C(0xbacead50d7f6acc2), C(0x6bc63c666b5100e2), C(0xe0b056f1821752af), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf8eadfaca72b6370), C(0x2ff5cd51c2eb1ed4), C(0x4a16ce6ad0c8515a), C(0x7823b92e0863cb7e), C(0xd6db820d0e150186), C(0xaa24dc2a9573d5fe), C(0xeb136daa89da5110), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xafcba310d3f470c3), C(0x23ecd39cf152688c), C(0x16829a08db73ebb6), C(0x756b3a471adbdea5), C(0xd24646c267ea1f2), C(0xe81f4c4a1989036a), C(0xd0f8db365f9d7e00), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x546ad0dbfb9f685a), C(0xe26387c036f6cbf8), C(0x5150a1967b016512), C(0xd80611a6e84c890e), C(0xf05ba9be87e230fc), C(0xf0c6624c4b098fd3), C(0x1bae2053e41fa4d9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1ea4b6086d96b2f4), C(0x8fba4a5ee7a4f9a1), C(0xd9ce86c50368f487), C(0xd20b16270e4be8e2), C(0xdfa63433183a5d9a), C(0x3a9c8ed5a399d0a9), C(0x951b8d084691d4e4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc14a1b73f3773c8d), C(0x66b2d5049fc7acf5), C(0x23c4ab45908b1748), C(0x7a9c7215c9ea097d), C(0x143994542aa22f70), C(0x9c0178848321c97a), C(0x9d934f814f4d6a3c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa7e9b9f09d6bf7b7), C(0x32367c98d0f88cc4), C(0x4ee1d87a4c49c5a6), C(0x3b44c5438b9ff383), C(0x9105b5e1500c9647), C(0x42b192d71f414b7a), C(0x79692cef44fa0206), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb4927ae39d53bc5b), C(0xa6c6866d19840c22), C(0x92b6d63a63813e2b), C(0x3f3a64f52594b108), C(0xdab4250729d6b9d9), C(0xfa5d6461e768dda2), C(0xcb3ce74e8ec4f906), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf3fd396b782f2a99), C(0x2f665744d98ffe47), C(0x81563b58d0920a8a), C(0x42a19c6c32827264), C(0xf9dc632849387dc3), C(0xfecfe11e13a2bdb4), C(0x6c4fa0273d7db08c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8264c5cfcbffd0d9), C(0xdaaf578999048a43), C(0xd1bc39ac5de6f891), C(0xac67bdf0bb8d9f74), C(0xa0d09bacb7e76c0), C(0x7bb50ffc9fac74b3), C(0x477e70ab2b347db2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8da705e357584a5), C(0xe13ce37fda1ed7e4), C(0x5e4c185e3cff62f8), C(0x3023524ef4e8258e), C(0xb19af60cdb67b5d9), C(0x5f97b9750e365411), C(0xe8cde7f93af49a3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1e58797a3777956e), C(0x150c9d0cba45eefd), C(0xd9bef89fc1700ad7), C(0x4924076cbd02bb1e), C(0x2495980a4c9235d3), C(0xda90fa7c28c37478), C(0x5e9a2eafc670a88a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe67dc878f9fd0b48), C(0x88074dce3568a650), C(0x720a3a380e613bd3), C(0x6a11a71062e89ad7), C(0xca9b3eec85c6c145), C(0xf4b70a971855e732), C(0x40c7695aa3662afd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x43c18dc2183a85d5), C(0x35bcb5695b12d7fa), C(0xaacaa54f2dd24278), C(0x7b1feddee46d2ba), C(0x7aab0e0158f3b6e1), C(0xcd6da30530f3ea89), C(0xb7f8b9a704e6cea1), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2684ddfe1bff5138), C(0xcf162753dc0cb180), C(0xd46dc59006960658), C(0x9f341c8724a85fa), C(0xc8d4b4b9b45cbe88), C(0x39bf5e5fec82dcca), C(0x8ade56388901a619), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc36fcbd65f684b36), C(0x8ed97ca915e6d0bb), C(0x6f4d23e2f86ca1bf), C(0xc329bdb4844f36b6), C(0x3376eab257f59f92), C(0x82f3503f636aef1), C(0x5f78a282378b6bb0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xfcee317c5dd8e661), C(0x8dc1d0d5859b3506), C(0xdef6cc9601da20d7), C(0xfd39c446cb170097), C(0xae73bfa47b001bf), C(0xa644aff716928297), C(0xdd46aee73824b4ed), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5fbe0cc098e6253c), C(0x2f42d10f6d5052d4), C(0xff61398e9dcc3e7c), C(0x8a621e89c6e83bee), C(0xbb3df752ed2694ae), C(0xfb53ab03b9ad0855), C(0x3664026c8fc669d7), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4640e5467e2f959a), C(0x36093aa74487aecf), C(0xd164c7865659ae98), C(0x27fd0c01df4a9c27), C(0x40260438c2466d0), C(0x8e959533e35a766), C(0x347b7c22b75ae65f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x75a34298f73e9fca), C(0xff8e75b212dd392e), C(0x3afb8d169a8e019d), C(0x56f4c707cd4840a2), C(0xb68e5ef023989468), C(0x89ef1afca81f7de8), C(0xb1857db11985d296), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1e12b3799eddf003), C(0x67c21d7ecb76b350), C(0x660ceab6bcbc1edd), C(0x2e368afca726f268), C(0x2a76d35deb67e18d), C(0x380fad1e288d57e5), C(0xbf7c7e8ef0e3b83a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa048588e4cc6625), C(0x95dd0cdf4bf74697), C(0x8f4a977b4e0c8b62), C(0xb1ba563883cfa53b), C(0xd89ec69bf787188a), C(0x6b9406ead64079bf), C(0x11b28e20a573b7bd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7c37833af2b3509c), C(0xcbe4f7c6c28ceb7b), C(0x79edb6891ea45050), C(0x1b6701e3e342b1a1), C(0x38fc9980ef01ea41), C(0x236542255b2ad8d9), C(0x595d201a2c19d5bc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x10ceeb033d30d7fc), C(0xc2487e1df5c7f59a), C(0x5e998a919e50d6a9), C(0xe39cfa7607d81faa), C(0xe823f27b3c488883), C(0x6189dfba34ed656c), C(0x91658f95836e5206), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf7802b66e6485ce8), C(0xf215c799a475d715), C(0xbc2d80a71f2b7bf4), C(0xe5df8bd4af890217), C(0xa75799d36c309ae7), C(0xa674d85812c7cf6), C(0x63538c0351049940), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3b7b4b79e9530d39), C(0xf7412431374c78bc), C(0xec6c5d44efe184b8), C(0x445aa9c46792ce64), C(0x6f9cdad707b71b28), C(0xbdd7353230eb2b38), C(0xfad31fced7abade5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xaa463be8aa9397f9), C(0xc2fc420945c4110f), C(0x8dd5a41fd502ce93), C(0x450e6c4ff53debce), C(0xb5e917d78f0cafb7), C(0x434e824cb3e0cd11), C(0x431a4d382e39d16e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc979a7e1e48cc1cf), C(0x9435e19d02cdfc5b), C(0x593e99cee88e282c), C(0x1f33ab45c93068ff), C(0xcac5bf84c51c2460), C(0x7c6bf01c60436075), C(0xfa55161e7d9030b2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9a4c3956f7076999), C(0xafc6ca28d864f66f), C(0xac2d7a8a41c1efd2), C(0x45b0669c4147730b), C(0x6d36b7066a9e2fb4), C(0xc3f40a2f40b3b213), C(0x6a784de68794492d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x48e30a1ea00756ec), C(0xc3902be98a47c18a), C(0xf50ec4db63014e93), C(0xa6add4bec1720542), C(0xea03976e5cc1a86), C(0xfea3af64a413d0b2), C(0xd64d1810e83520fe), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x82e03f3aa98aff75), C(0xa0292ce77b71e2ce), C(0x7d577756324c6bd6), C(0x4c958567626d38b9), C(0x29e371e785317087), C(0x1ce621fd700fe396), C(0x686450d7a346878a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe1e5c15418a7eeb5), C(0xd566cf8d17a1a9c7), C(0x7f9d80e52fb3d5f7), C(0xe5cd49142616aa69), C(0x381106b646d128d1), C(0x7cc06361b86d0559), C(0x119b617a8c2be199), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9884a8d98c51bae1), C(0x3cb348ea1b3b54bc), C(0x152395417bb7ad30), C(0xed2fb9c7ced0af65), C(0xa8f4a145e9da5beb), C(0xdeb85613995c06ed), C(0xcbe1d957485a3ccd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x74520d7ecc269ddd), C(0xf92219f5fe0a6758), C(0xee26f3756ceadd13), C(0x998bd660601f89e8), C(0x53268902d428ff32), C(0x84f80c832d71979c), C(0x229310f3ffbbf4c6), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc39671165e8b2bd9), C(0xb6c216f6f70a4132), C(0x78c47b6653f9ef0e), C(0xc035ce3bc0818dbf), C(0x3c2f0333e7b7dcea), C(0x8dddfbab930f6494), C(0x2ccf4b08f5d417a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3b7282fb96edc5f8), C(0xe566ebeb8b834ecb), C(0xd422702f7053db59), C(0x60dcc9edbf2d2956), C(0xaa1e49bcda347c20), C(0xfa722d4f243b4964), C(0x25f15800bffdd122), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5cc607671cc54a0e), C(0xc4ba56cea4b9772a), C(0xc831d7b72fc7d3ee), C(0xd3da99c67e5df166), C(0x6ca2dae0b688e65a), C(0xa96dcc4d1f4782a7), C(0x102b62a82309dde5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x34bb51ab07b5f566), C(0x17f6266960e96415), C(0x282ca656ad1b652c), C(0xdec0691980d9c025), C(0x7ce80f93472ee4a5), C(0x2d553ffbff3be99d), C(0xc91c4ee0cb563182), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x72622f2b10de6d6b), C(0x2767d831a05ff08a), C(0x33d5002e23da6ff2), C(0x54569b470cfed9eb), C(0x3b1ae2a607451568), C(0xa5c550166b3a142b), C(0x2f482b4e35327287), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb624e9707ea879f), C(0x912c7ccd5c4c8327), C(0xaa975b6fbbfa8a20), C(0x71565fb68cc48592), C(0xea86f9ff62cac9b0), C(0x9653efeed5897681), C(0xf5367ff83e9ebbb3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x89eececb3a3883a4), C(0xb607c348c4742672), C(0x96565545dd9205f9), C(0x74565f3e108efa3c), C(0x29532b1d9c27ce0f), C(0xc0ca86b360746e96), C(0xaa679cc066a8040b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbd23872256dbf216), C(0x8913bded7e0ef012), C(0x6a1fb4504ab0c356), C(0xef1b4e6e851c8139), C(0xb11db03ae4debfd5), C(0x814aadfacd217f1d), C(0x2754e3def1c405a9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x432dc78e38b2b88c), C(0x339aef22ae2840f1), C(0xa9b553de119dd498), C(0xec2bdbf3b7569cf0), C(0x46a08ad7635bc4b9), C(0xfcc09198bb90bf9f), C(0xc5e077e41a65ba91), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x70c3594fcc5804ee), C(0x28e270597aee14d1), C(0x6f277e69c5feade3), C(0x510f304b3b59322a), C(0x4b9c45919dc0b3d6), C(0x308bd616d5460239), C(0x4fd33269f76783ea), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x64e124d478c1f963), C(0xcbd254de16ad2e91), C(0x2f07aa0b9f9502e7), C(0xc406f98356586912), C(0xaed868be6ada8078), C(0x24dc06833bf193a9), C(0x3c23308ba8e99d7e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xab74ee59c2f9fd97), C(0xbca2dbba9276c039), C(0x2ec3d8c1d7f98f1f), C(0x406293ac4639aa79), C(0x42dc9622f9ed209a), C(0x301b11bf8a4d8ce8), C(0x73126fd45ab75de9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5fe253681540a9f9), C(0xfb27f3755950447d), C(0x9b2f2b84495d27ff), C(0xb2bdfa7552f0eeae), C(0xeddc2c0855761fe3), C(0x48f4ab74a35e95f2), C(0xcc1afcfd99a180e7), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x131598d20f7177d2), C(0x4dfa1ed83fc96ec5), C(0x8a8cf453d42a9703), C(0x807a4a531dc44c53), C(0xac569a40e2c6d83a), C(0xf2bc948cc4fc027c), C(0x8a8000c6066772a3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc2865c947d4a3767), C(0x47454b1e8b9bfbb9), C(0xe0d26745d5e3100f), C(0xd8b7a5ad22fe9c66), C(0x951248494fc50d8b), C(0x178b4059e1a0afe5), C(0x6e2c96b7f58e5178), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd6e457ce8f99dac7), C(0xf5887d014261a774), C(0xc0b87384ce98da79), C(0xd2a5b45192785dbe), C(0x26f3b91ba038588c), C(0x5f3b792b22f07297), C(0xfd64061f8be86811), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7085f9a27f5c35f5), C(0x538e7930fad7e653), C(0xa0028f1e3c3317f0), C(0xab4dea70451fed36), C(0xa319425d4e740342), C(0x9fc3c4764037c3c9), C(0x2890c42fc0d972cf), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x40fde0167608e3c3), C(0x4bc14d90dd50da02), C(0x2b85e31097978d87), C(0x804139c438c8d74d), C(0xd3c1f9ed22af3414), C(0xf169e1f0b835279d), C(0x7498e432f9619b27), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe6690fd7fedd557d), C(0x1b2ecb79f8ff3d72), C(0x779ef63892ce8777), C(0xcc287c8eb296265), C(0x660d43c157382a4f), C(0xaa6cb5c4bafae741), C(0x739699951ca8c713), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa48744daddaf84c9), C(0xe4577cea3c5abc39), C(0x5a52424e98ddd071), C(0x9e1b8b7e452c588b), C(0x7ecf0a7a00893386), C(0xdd86c4d4cb6258e2), C(0xefa9857afd046c7f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6b13a0100dedec79), C(0x6b90824ea06d6932), C(0xef3b744321d4cdd8), C(0x7cb7925c3a184c78), C(0x44ac1e5e256787b6), C(0x96d5c91970f2cb12), C(0x40fd28c43506c95d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8b2a0d2702916340), C(0xb1d422021f5263a6), C(0x4e6b643f71d8251b), C(0xa83ed5ce742683f5), C(0xe007bcc532d794c8), C(0x1fbe30982e78e6f0), C(0xa460a15dcf327e44), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x720b6ffba2808792), C(0x5a1eeb32e165345b), C(0xd94a75b38cbfb4cf), C(0x879f5a90d7acc458), C(0x42ffaf4b159b454), C(0xf1bf910d44bd84cb), C(0xb32c24c6a40272), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5b9450322bc68263), C(0x19d28db5b1d30e00), C(0x9803fd62efa15d50), C(0xe87d8247a4150e38), C(0xe6963ebc11f4f246), C(0x30b078e76b0214e2), C(0x42954e6ad721b920), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe195aa10d7cfe8f5), C(0x9d95d76ab979902), C(0x157a8c5a2bd91648), C(0x39a41ecf8b3171cf), C(0x7c7da0e00f19072), C(0xaa0fe8d12f808f83), C(0x443e31d70873bb6b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4aef776746299e4e), C(0xd0a4b408cf306ba0), C(0x1f143ad71cf8a24f), C(0xf1e08906155b04c0), C(0xc787e8e3cb94bd54), C(0x18e002679217c405), C(0xbd6d66e85332ae9f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc7846db10d5b0801), C(0xf633cb6792a68091), C(0x7085c4319a78feb3), C(0x4bc29b36ff94405), C(0x4e8d8a9e86c00ba8), C(0x89b563996d3a0b78), C(0x39b02413b23c3f08), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x468758d7c6dbea94), C(0x674a74eeaf00dbc5), C(0xd33a0ddcaf78c6b2), C(0x87dec3bd8bb1105b), C(0xad212ca93bdd2655), C(0x9723a9f4c08ad93a), C(0x5309596f48ab456b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3f161ad3ccda8159), C(0x31cbf83a3b530240), C(0x76287b00836006e4), C(0xf9bb65b54081f31f), C(0x152492732c55c78e), C(0x4f3db1c53eca2952), C(0xd24d69b3e9ef10f3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x21445292adbf3c2a), C(0xe2f3840b2aa15aea), C(0x812f5a840916e28c), C(0x7b06f5b212130dad), C(0x9e5372da69654871), C(0x37b2cbdd973a3ac9), C(0x7b3223cd9c9497be), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6fa4782a0971ddb3), C(0x9cab66441fa0f232), C(0xed6dc7aa143b3c52), C(0x5abbfa60974dede9), C(0x54f5af2e26435486), C(0x2b9f07f93a6c25b9), C(0x96f24ede2bdc0718), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7fdd5b47ace106db), C(0xa5eb83de169efbdf), C(0x5180932a48b8613d), C(0xd37a588f65df8483), C(0x95ab6f21206b720d), C(0xc77dd1f881df2c54), C(0x62eac298ec226dc3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc8cdb83ec1dda280), C(0x14befeb18c713b61), C(0x71171b5c6d90629e), C(0xbc0013b518146b1), C(0x5608cd39a26e2a17), C(0xd92f7ba9928a4ffe), C(0x53f56babdcae96a6), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc73d49846863238f), C(0x367a591a5dbe36bf), C(0x83028fb3929c300a), C(0xb4a028f94bb8694), C(0x337b48e93e007d93), C(0x7bab08fdd26ba0a4), C(0x7587743c18fe2475), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbf929720d27d4e74), C(0xe08e048b4c1b938d), C(0x8733fb996da6284a), C(0x460773a58d947074), C(0x3cb4720c4e3c9db0), C(0xf4f12a5b1ac11f29), C(0x7db8bad81249dee4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x316385a97273d36c), C(0x689329d08b30496f), C(0x33c2053bda670550), C(0x73d889808efeea48), C(0x362c95419f53fba0), C(0x8257a30062cb66f), C(0x6786f9b2dc1ff18a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa35d327b6158a633), C(0xdeba8db534b3b9fb), C(0x792c657844290eed), C(0x5a1bf5910ac5b207), C(0x8eb3a88f4bfeeb47), C(0x9e89ece0712db1c0), C(0x101d8274a711a54b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3091bf084dcca587), C(0xec98b7e268cbe95a), C(0x4eda4c50a24f3d41), C(0x38c0fbf61e9ebf87), C(0x298e3f20be872352), C(0x2140cec706b9d406), C(0x7b22429b131e9c72), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9a77f98e1bbce2cc), C(0xe66c9812b7e116c2), C(0x9216eac2bdaa1fbb), C(0x834c803b0e7b07fe), C(0x745287427a0dea5c), C(0x5ac8ca76a357eb1b), C(0x32b58308625661fb), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x43cb1634a58af4a5), C(0xad41864d25a9dedd), C(0x1dbbda7f2d79d2a8), C(0x4a0747dedfccfb03), C(0x2ce3ddcdb423f3de), C(0x4ad276b249a5d5dd), C(0x549a22a17c0cde12), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb3feb31e7021d1a3), C(0xf77d22f7e2d35ceb), C(0x1c51da411b82f89c), C(0xd61146c9f2c64d04), C(0x4fe0c67064eda9f1), C(0x8ebc520c227206fe), C(0xda3f861490f5d291), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xaf3cab5151956f32), C(0x368a869c04453884), C(0xb8ac4f4a15926e27), C(0x6201341ac742d736), C(0xb96d17e1c467f2b5), C(0xe42693d5b34e63ab), C(0x2f4ef2be67f62104), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x104c71fd5a44a02e), C(0x9ea3a815b5b10b6a), C(0x81d9e5892438948b), C(0xcf51207ad3d9feda), C(0x81bf069a1afdd68), C(0x37e9dfd950e7b692), C(0x80673be6a7888b87), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7615abf4800d03de), C(0x3fbf88dba9021e20), C(0xf94c95c95d28d1c1), C(0x5a031558c7eefc15), C(0x8bc38c58f9e39b14), C(0x4438bae88ae28bf9), C(0xaa7eae72c9244a0d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2c54f6cab59038f0), C(0x981ef1f94d2d824f), C(0xc84d1476c2cfb3e4), C(0xd6aaa399a0834d4c), C(0xcb3b491b31150982), C(0xbfe279aed5cb4bc8), C(0x2a62508a467a22ff), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9660da1239deea6e), C(0xf60422e527d9863f), C(0xaf8371a3e5dd29a7), C(0x526b279848ce5ea3), C(0x5791d3f9afccd1ff), C(0x679c204ad3d9e766), C(0xb28e788878488dc1), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf69cfc86f522ec99), C(0x36be168cf6a167e8), C(0x1b09ccc92f7f8a3c), C(0xa7f7eaf8470fc07b), C(0x5bd419533de91276), C(0xede23fb9a251771), C(0xbd617f2643324590), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x97baf65105fc2685), C(0x638fd69ae4a0b881), C(0x90dd12767578057c), C(0x1032c6fe2cdd44cb), C(0xc57077534741f5d0), C(0xc7c1eec455217145), C(0x6adfdc6e07602d42), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x99df889c7e3b45ef), C(0x74a8d52d38deaa58), C(0x9d4de60f6980b894), C(0xfa00e9b866b954a1), C(0x36bc9ad6e9e94ced), C(0xfcd6da5e5fae833a), C(0x51ed3c41f87f9118), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf822e66918469e1d), C(0xadb1585d362485fc), C(0xe9bcedd28277901d), C(0x2f6ff1b1bf6c70a5), C(0x88ca571198019725), C(0x9841bf66d0462cd), C(0x79140c1c18536aeb), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1c2b96f94ec3addf), C(0x7dd1e7de52fae722), C(0x3fd087d70d3d1379), C(0x6a70f0bdd7b3d161), C(0x889bf55dfa6ac252), C(0xf6915c1562c7d82f), C(0xe4071d82a6dd71db), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5b0eee1eb438ba44), C(0x837e8f053b8df92), C(0x96a9566d665a01bd), C(0xa7810b71a02a9d51), C(0xaf403849304bad56), C(0xcdfd34ba7d7b03eb), C(0x5061812ce6c88499), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe140edd9491c8219), C(0x53c05140c7343562), C(0xc5eaeede1cc02b2c), C(0xb8c077da249cacf2), C(0x947ee3d8f79a2063), C(0x6de42ba8672b9640), C(0xd0bccdb72c51c18), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa43b3b1cafcc5179), C(0x9585f7a3735259bf), C(0x17fe8f5f41d9da0a), C(0xef34535e2b79889), C(0x1bfcad698ab4f8af), C(0x345b793ccfa93055), C(0x932160fe802ca975), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x83dfc41bff95a7e), C(0x78744a43f096eb39), C(0xe20949ec0319e6c3), C(0x98e2d802c1ca71ac), C(0x66a515e92a5fe4ae), C(0xe30e4b2372171bdf), C(0xf3db986c4156f3cb), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x53c8cca8585ee88b), C(0x471a2284a3ffd625), C(0x77d248379356c42f), C(0x856aa76713e0a34b), C(0xe78aed261854142f), C(0xe9cc71ae64e3f09e), C(0xbef634bc978bac31), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x58022d172edc977a), C(0x6840d36b2c902ab4), C(0x1971575ebcf95124), C(0x61237b5567b9d77f), C(0xeefa2a1e718fa37), C(0xed186122d71bcc9f), C(0x8620017ab5f3ba3b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa3835b1aa4cb303c), C(0xf870fcd77b9c071e), C(0xa56dd5cafe5abe24), C(0x80e057b3e6bc4277), C(0x1e8ae5d04c7c0f25), C(0xcb1a9e85de5e4b8d), C(0xd4d12afb67a27659), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x617b2e32ad11dd34), C(0x1b4b6b7b4f7d0666), C(0x6842f84c8992fb54), C(0xec51051eb6767dac), C(0xd7bd6fd9ff408414), C(0xea3040bc0c717ef8), C(0x7617ab400dfadbc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x33b7bee7e4424a18), C(0x1851732a22285037), C(0x355f95f09d27d3d6), C(0xa883e54b3868ce53), C(0x100a6a5bd79480d1), C(0x4f1cf4006e613b78), C(0x57c40c4db32bec3b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x893de34989d8236a), C(0x5e499786cade7880), C(0xd060a6e1bb81dfed), C(0x260c87edbb85731b), C(0x249e83d79b92cff3), C(0x4383a9236f8b5a2b), C(0x7bc1a64641d803a4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb158d322a0072315), C(0x24ccefc413d23302), C(0xe7f509c413975392), C(0x8c70cbdb9ccef69a), C(0xb4499ee47fe49a01), C(0xfd9bd8d397abcfa3), C(0x8ccf0004aa86b795), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5b2029e1ef27808d), C(0x94c7496316b6db04), C(0x8472b9f196d0b2ec), C(0xd5f47d8dbe8af661), C(0xe25368388c156618), C(0x8eccc7e4f3af3b51), C(0x381e54c3c8f1c7d0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xac9e5787db2f0335), C(0xd5ec72af78d37332), C(0xb4daded474f2df55), C(0x8863854124d5bcdb), C(0x3c3cf93aa88cf091), C(0xdbb71106cdbfea36), C(0x785239a742c6d26d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd21806e08aac3b7c), C(0xb450b7e770b216ee), C(0x987a7377bf634988), C(0x4c669adab9c2e6fb), C(0xfbfb5b46fda6e343), C(0x2e329a5be2c011b), C(0x73161c93331b14f9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbc9db4eea4997698), C(0x1236095365b1f3b5), C(0xe5b16b7e7e9fcc7a), C(0x63932710f6d1aa14), C(0x79d3852adadbf992), C(0xc46f0a7847f60c1d), C(0xaf1579c5797703cc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x67bdfa2af2feba27), C(0x463f49d88fe9ef4c), C(0xebc7ce72b30feca2), C(0x51b0a1a586b23426), C(0xb2dd8b8908425506), C(0x1c842a07abab30cd), C(0xcd8124176bac01ac), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x132ef8b26a1af44c), C(0x31d0570b0c9172e6), C(0xdfa6a38b5281d9ce), C(0xe669872a046eb92e), C(0x2b4e1d0a3f12aa85), C(0xf18c7fcf34d1df47), C(0xdfb043419ecf1fa9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x286163c54897c913), C(0x5296ee61f3aa836d), C(0x752341b4678f4b9b), C(0xeebb5e79fed4a57), C(0x181be485284dc229), C(0x65e9c1fd885aa932), C(0x354d4bc034ba8cbe), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xde8e77b45e031e8c), C(0x664b779ec74208ce), C(0xa1cfd23a083f2cfc), C(0x43ad050e96a213d9), C(0x6fec16412f85b3f2), C(0xc77f131cca38f761), C(0xc56ac3cf275be121), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x43991ac2dafb5583), C(0x2dd1fc0424ee9b3), C(0xb23ef52163a01c76), C(0x27162f4a83100f16), C(0x9795c377db26a7b3), C(0x429f935fba7a0e8e), C(0xe991298919233781), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2367c7674f9b4186), C(0xd74629f1b793f794), C(0x87d41cb421bf7ce6), C(0x584f04ed0a17afd2), C(0xcd03dd8c237e355b), C(0x80a832b0db0aeaf4), C(0x9c3bbfb275fe9ae0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4f0ccd886aeaa45e), C(0x19c6084bccd82397), C(0xecd5a9898d3f1f84), C(0x78751baea194bf90), C(0x9046ccb3aa41c895), C(0x4c3e06dc260108b1), C(0x7b638b432b07348a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3f93960447b0f41e), C(0x5af0021cd8c9190b), C(0xa6bb529625479c11), C(0x58c04fa3216cb06f), C(0x59352346e13a1e0d), C(0xfd3f257f00325a25), C(0x9654a1ba577d7e9d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x655c4693fea0ebd8), C(0xf50ed27abf3ee825), C(0xd7d122541c5d7ce9), C(0x808ac6db23d2eed1), C(0x5175a45d64db90de), C(0xb36c6c4e0635e97e), C(0x91398aca83518c2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xdc713b0c1df2e130), C(0x276b7a79bc6b0ced), C(0x72e523771a4d04a0), C(0x7422b3dc34b2d24b), C(0x7cebf518e224d3f4), C(0x18dee526144c3773), C(0x30a7fbac39654376), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1da8742a924ad91d), C(0xe4c71d72610e20c4), C(0x71478d6343b645c3), C(0x625a064671caf18c), C(0x63937950fcb40747), C(0x972f4978becff208), C(0x73e7c92ed536728f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa65e93532318d21f), C(0x55c1181234a06fa6), C(0xd173f8f5583d911a), C(0x7db04180e3d35ff8), C(0x6f320c7c9e9b4a7c), C(0x33173bc5f2e3126d), C(0x8dc4edca4f0e75bf), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd33358c9d79d375c), C(0x17a29d209cf46ede), C(0xba4fdc94b2058549), C(0x401ea4f2f4153e7), C(0x37a5bce6c5293893), C(0xfc49fdea809e6a5f), C(0x8d11a0cf694bcd13), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x65ecc61fee720200), C(0x9b3bdd718af21376), C(0xc5822d258b317021), C(0x334b96c19026238e), C(0x7a53c4d15f609bd9), C(0xa24089e51527eef8), C(0x93db966ce70acd4f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6eaf317841fb2460), C(0x8280182820128065), C(0x95c825fcca607219), C(0x58ccf9673d904af3), C(0xbfedaa65fb40e6a), C(0xc45db1f558aa82f4), C(0x97c2ced4c2b4ab74), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3b15db8385bd540), C(0x6f0b1039aed94e1e), C(0xcae94c7f7a26eac4), C(0x41173b7d6b9288f0), C(0x172219e1c0f497e6), C(0xd402acbff69d734d), C(0xcfdd50534210adb8), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4d5b182ed7504c53), C(0xdf9bc0ce68d47f20), C(0x7295d33e7755e364), C(0x41b08efe0ddbe05c), C(0x3d3050d504800767), C(0x176d142d00a57fd5), C(0xa2555ad65af9306a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd4148382ed0c1a7e), C(0xb14cb62bea7ac5f2), C(0xffdf43df3ac7bc00), C(0x96f811cd8969e73d), C(0x7d97d002cfd1227a), C(0xfec3f26f85392330), C(0x873830deaedc5c70), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf941f66ef3b3adeb), C(0x77cfe23a5fa3bea), C(0xaa1f4494f8b91364), C(0x62c1e3cf3f15cf6c), C(0xb0c1eb21e2a8c03f), C(0x7e108b3a8f5ec19f), C(0x1e308430746f3cb1), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc4b133d694367236), C(0x6fb7ae799d346afb), C(0x67ad79a760ae67ad), C(0x2a145dd3c792d176), C(0xe1fe89ffd04e5da3), C(0xc4da2b0ed7d58249), C(0xb3e53011bf0871b9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe891cf68fd2bd413), C(0xd9bb1af1cb4d5db8), C(0xd129d95cb295b5c8), C(0x6701ce79dccf629c), C(0x48b7f1bbc78e99b7), C(0x40db60c51e9b8163), C(0x16de75584883f55e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x170066a856def440), C(0x9dffa440da4d7422), C(0x1fb0c45005f59f38), C(0xf9e0b220a4cc3ae3), C(0x10d6699ed3a4632f), C(0x760daefee72daf58), C(0xe9083d145ab2e0bc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x799ddba2fa54a6ef), C(0xe97e8f8f25c9f229), C(0x798218d1c4282ff3), C(0x100c06ff2b6fa888), C(0xbf32d2f55480c3b9), C(0xb7b777930cd4bfb5), C(0xa77f5d1fc82e6f6d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x884ac60dfb4dd572), C(0x11f66c9202ceff52), C(0x17499a442342f26f), C(0x9c70be20d262a3a0), C(0x2684adc62f176a50), C(0x6afadb8b1a75e546), C(0x3afc5789c5af9d7a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x204f1439f7fe19d), C(0x1a6adc9f21ab233b), C(0x85c15c91074f47c2), C(0xa7e43f22f693f0ab), C(0x64831b7dd7307ca6), C(0xf8272c673bda5d08), C(0xeeb94e53f2a14082), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x55636cd0d4b16c04), C(0xe5604c4e9276b03c), C(0x83056280084d66fe), C(0x1bf3971cc8e712d), C(0xcc256d2baabe8ab1), C(0x204a53ad0b713713), C(0x7e63318b704ac9dd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf9f64d26f7c8c01d), C(0x9da658b75222cdeb), C(0xbaffdc245a52c3b8), C(0x7e23d5127db80ff6), C(0x7005f02670bd673b), C(0xca806427f80d8854), C(0xa4ec224cb8f0cc6), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x18ec9cd25fa8119d), C(0x554102d88213f5bd), C(0xe39dd9dcd9c6d24d), C(0x7f8d11a8fc15894e), C(0xb2fe009f5acd820e), C(0x75ed64126bb12911), C(0xe521d97a3b2288f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x63cde81bc96b4d6d), C(0xe343ece780b0e7a), C(0x3b733bdb410e2d3), C(0x9d5022b97e51139a), C(0x5e06b3d4187b1fc0), C(0x44d63c9c8d363e73), C(0x81046b5941ed0960), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9cb75c87f14ceed9), C(0x9b3c0e13e9027232), C(0xc77fa6a661377d23), C(0x96dee94ab0958bba), C(0x5e8f414669d94369), C(0xee690496a6baf1fc), C(0xaafe74baabc01c56), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa4117387529414c7), C(0xeb14672cbf3eb81c), C(0x41748712f045f98b), C(0xf9734d66d84b3de9), C(0x15ec4f373a48f7fa), C(0x523ffc8d341e8b7), C(0x9f10cfb6f3927f30), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x772a7376f56a0090), C(0xc8dbbd6b446de59e), C(0xe204c2d56f7873ed), C(0x3850c4f19307486a), C(0xca0e073417d3b866), C(0xfc3929e65c2a4e7d), C(0x214774db5ead288c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbbddfc834e0cad4b), C(0x269040ec2bf67759), C(0xc0f0b69b55d4a102), C(0xc9399463297b4910), C(0x330057f8dbec3aa6), C(0x23695fdf0cc4de7a), C(0x9aa0dda74cb06516), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbe396d864e19a93c), C(0x63c3a33a58c673ef), C(0xed581cead350466b), C(0xee17af910951df8), C(0x1e95be7050c18ed9), C(0x4bb8288ed7ea8ea5), C(0xc7b5d22fd2db5bca), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7098302f4b41637e), C(0x8d2d305464240bfd), C(0xe96dd76823e6d604), C(0x14023b6c2f6d77f0), C(0x5ef44110e4625b39), C(0x8287bfb7cfff5278), C(0x1365fc6bbd09f931), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1cf48de0dd627a04), C(0xf33a18f5b7706bd2), C(0xf42699a833e2dcfc), C(0x9d173eb3f4df3c3c), C(0xa42f4d6257186336), C(0x3fc50520484d054), C(0x7b53dbfa9e033583), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd789d2adbc232f40), C(0x8cd5d3d39ef7108e), C(0xebb2e0810a61c118), C(0xe91166bf8596d49e), C(0xee7038b517b2d113), C(0xb710fdad0e8963be), C(0x85a664af226c21be), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2494f64ad14c0de9), C(0x66202e9f9b552c1b), C(0xba704fbe82ff6c30), C(0x320a1fe6262695a5), C(0x3dfdf1cfbb185765), C(0x6fff46fa1f314a4), C(0xcd39f69512847bc5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa4d5907f8da27d23), C(0xc22d8413497ad864), C(0xca32d49cc6a02cf3), C(0x190fc8fcf0a491d3), C(0xad97d786e3da3ad8), C(0x30c3ce2265fa97d0), C(0xbf0b97ec9afeb926), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3ed3ca4a7f7f58f7), C(0x33e65c2f5818c873), C(0xf0ce287d6353bf5e), C(0x1b6f34538beff4d3), C(0x3a3afa3fa25cdd95), C(0xfe45852346bb2c5b), C(0xeefbf422b4516e81), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x95d59381f9ac7ecd), C(0xd72c56d6ea3bac4a), C(0xc897016da43933d5), C(0x82752018b97573c1), C(0x32185472c1f99edf), C(0xf4675061253439f7), C(0x41d00e823c47ff59), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc502e5c7e7565602), C(0xe582dbb67fdcdcc6), C(0x456a76f7b8e3e664), C(0x8807bfa60b10aba0), C(0x1556cd5e29e43419), C(0x3582aa2014befce2), C(0x4a92bd41e14487a9), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x83a7b1eb536d182b), C(0xec1d9ac0ec0f4e40), C(0xf025d18aae2dac26), C(0x3a80953eb090e014), C(0xd3e89d4323d5de6d), C(0xc1907bb9794a9890), C(0x2e5832dae7816503), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4c6f3ed4f750e475), C(0x5cc52e1a502f3238), C(0x6c50d400281b9e10), C(0xa01c7d96d49b3f65), C(0x6d8aafadaf0c81f), C(0xb13f37c3f0e42022), C(0xd2f498ce7efc1e5d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2bf900474df2148f), C(0xfe66283eef7f11d6), C(0x396aea9faad76eeb), C(0xe1304147104f04cf), C(0xecd289597aa47970), C(0x5493e823f3091de9), C(0x59019e4dcffe4299), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x15977d178f3686c), C(0xbd4ac1ec7122a2c4), C(0x1c0f7da89e71d818), C(0x99bbd042a39d5631), C(0x7f9ac288a488b5d0), C(0xa285cca06bf5faae), C(0xa789c39564df1d18), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2ceac71488c8abef), C(0x35d30a7c89f5dc8c), C(0x14bf5ceb06edc93e), C(0x9ba08c30d1567e5c), C(0xa1eccd705e24ec37), C(0xb47749eefa75ef99), C(0x7a62f0237cf02f1e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9c8106bddd57df2), C(0x1e6f04161fb31572), C(0x5b58a315c61f3f6a), C(0xf7cff9d97f92d2c6), C(0x65db7f7a131f1c94), C(0xf9587e5b26073810), C(0x13f29a82e559b126), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8d2ad91d07da6228), C(0x9ffa877455e4aa7f), C(0x10479a69f55da0d6), C(0x3b852de9c74a84ee), C(0x8eacf7e1c2fe20ac), C(0xa690163b925dd001), C(0xeed3a1b49ad7fbbd), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x151ca7e559d107ea), C(0x8aa73fcb41287428), C(0xef95cb4dd0fc40bb), C(0x319876fc2d2aa105), C(0xc962292fc6df879f), C(0x1f1814ca73b99ab5), C(0x596942b5857ebec4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1d5569440c345122), C(0x579b663380a10a3c), C(0xfeed5ff4329135bb), C(0x4a0fbf7f7e677fcc), C(0x4e1d74cfdb06c30b), C(0x4c38806477ec4456), C(0x48c228e8366d3f78), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb0d6c01aaa2de542), C(0x7a2e6a216b0b8478), C(0xabef48df3524ec19), C(0x9270d5f585fb2e21), C(0x7fcca364d4230e36), C(0xf947ec047f5e1e7e), C(0x5b6b5edb12fc801e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x69a449fa00de6a79), C(0x5bfc50cfbb6888c5), C(0xab266dbcfa2af154), C(0x5164406923ad24de), C(0xa7e8189bf1af223e), C(0xb43e0625bec5a5), C(0xbfb5b7f4f9e19cc6), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xcaa8062e3fbbb7b3), C(0xd3355d19d1811ec3), C(0x66ca010b5f872aba), C(0x1cd04ae709d23c85), C(0x71a2ad83debd6d1), C(0xf8c864f7ed7546f1), C(0xa53bcec522b02457), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x22f0de7c1cd5bfa0), C(0x8bd50ce5180d1f20), C(0xf34491fe7c987cb9), C(0xcb4c22b0e31adbae), C(0x5e42a7e5aee0bbb2), C(0xb78ca4da20bb8b47), C(0xfe89d33c8b876e9a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc66c28b40515996b), C(0x6f6f5b71f263f0f1), C(0x78cba0e7a232f349), C(0xc8a5490a3bb1573b), C(0x79d269de8bc29c3a), C(0x646e7d85143d228), C(0xe87448cdcf9d66e4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xccd4d37712f19e9f), C(0xe94ac54dc5903759), C(0x7b321166c18c6176), C(0x24535314244b7330), C(0xc89a22a4007ea1d8), C(0xe5d0ab7de64069d9), C(0xcdc548c002993e85), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4b2af9244d1c3a29), C(0x816186c635f6e5e5), C(0xa5ecf748dd9a4dcc), C(0x500bff0f2d0ce317), C(0xb353b5d20a7bccd4), C(0x2231f510bf2dd8d7), C(0x291a42aec2551270), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8bb2928cae20007f), C(0xca45e9108aa731d), C(0xac86a408946655f8), C(0xff86daf78607c033), C(0xfe6b88eed2ac0e9a), C(0x29794de866eeed6), C(0x2638b96181f9e332), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3573397bd4bec419), C(0x4daca2cec69e9f8b), C(0xddf59b60358edf53), C(0xcda340ea129ae7d4), C(0xcefe209561023c51), C(0x3a0dc7e70d015b41), C(0xa566caeda87afc3b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x82297f8ddec5c381), C(0x7783128b39eba8f2), C(0x48332d7a26e9fddd), C(0x199bc68457698731), C(0x92212c6630522124), C(0xedfc8b0cbe3fab0f), C(0xc41454201541567f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3cf70f6203baddaa), C(0x6c02d1f6faf9f909), C(0xb858f32b01eb298c), C(0xcf15dc0c3ebd8e28), C(0xb7c8fd6b297b25b2), C(0x3e57a9c015a6caac), C(0xbce97c88db4c44fb), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7890a569eb2ea772), C(0xa074e9a5ed63e57b), C(0xe5a32c23f6f67c6a), C(0x446b99785ca87df1), C(0x60c29fef2146fb9c), C(0xd22f7a3f46aec10f), C(0xcbab77eb46cc43d0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x254ac030cadcdc14), C(0xe50aa7e239a22eb0), C(0xcc85d892d1af3d9e), C(0xd50850453fa6651f), C(0xab7a36f4758b2215), C(0x35008ea5e18ae602), C(0x4b0cead88935d49b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x939e1dfd562a6981), C(0xa09335ab24c30193), C(0x7834a74d024adc32), C(0x72a98a3030d22550), C(0xf09c825e5c7c04b7), C(0xef2b78d033ff9cc), C(0xfbc4febf8f3ea589), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x18191c1a357bfeb6), C(0xc29e4e6f7bbb967f), C(0xc9dc2ed80945c8c7), C(0x5d8b0f3dd89a8b0c), C(0x1a43985acb3a7d50), C(0x1d0dede22c8e58d9), C(0x781cb48e6c9e964f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7a9db4f50f213fa9), C(0xb34a14db788ae496), C(0xc83c1a90f6718d94), C(0xf319edcaf1b23c5a), C(0x9585fb122b8c8989), C(0xb461bedc029bb60f), C(0xbe801390ce7453dc), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x808f618bfa395418), C(0xdbdbc68430a1b714), C(0x31c0c4c1612fefa6), C(0x9aa99ef81ac4dafa), C(0x4f255d17d83cf70b), C(0x8c77ed178655ac1e), C(0x2e074bf2d8d18a08), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3194b00cd82b5b35), C(0x6367748d8f89c37), C(0x1b8d71b40d656e55), C(0xb2cd3a5faba3a90f), C(0xd056eefce9c27e51), C(0x462ee8d7ecab34f1), C(0x6c2b8fc186ff5e02), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa2c5d544e7a5e908), C(0x8e52d92e13f6cfb6), C(0x5b272ba78306c66b), C(0xc8ab2f7b09b1b21f), C(0x4da41552813cd9a1), C(0x6ccefa934832edee), C(0x59d87c4c949cd0ae), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x45cc30d80cecd783), C(0x599caf75bf03555a), C(0xb947d88f7377e2d5), C(0xe5906c916ad3989b), C(0x9603034c10d98ab9), C(0xb7d0fe57a70a6c59), C(0xa5177e4ee6ae2552), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6ae17204aac7b4e3), C(0x1ba2e4ae9b8b6387), C(0xbd90222e8700c2b8), C(0x8811a8980f471c15), C(0x92adad7ce8eb7e5c), C(0xacb058bad34deff4), C(0xbfe2ba89a7d773ad), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb5539d23f4c5d7f5), C(0x1242146bb85cf446), C(0xf804110b944ce9e7), C(0xb472b50e27d1e06b), C(0xab215f4721a7366c), C(0xcda2aed65ffb8124), C(0x1c2d88370353e208), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x825f26c1fff59ffb), C(0xfa8d944735bd0280), C(0x9d10a7c0b29e743e), C(0x2676c70f75f1c90b), C(0x6ebee46303b577d), C(0x3bb1c25892ec4519), C(0x62372ca16f756025), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x86bdd42a12554204), C(0x7b511c7c41915e7f), C(0x59e88745055b07a4), C(0xb8a2ff293ff6a169), C(0xd23b113b9a29f), C(0x44d37484a10cd134), C(0x7cabdab7e19fcbd5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2cb0a5225b886f60), C(0x5e40fead4f8b68af), C(0xec07e360a55c1ab8), C(0x1a9a1d50ba8747b3), C(0xfa33927aa2589e0), C(0x64e857e411303d8e), C(0xec65e619dd9a3ff), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x408125504020e0f6), C(0xde3a6a8f46b12fdc), C(0x9a3cd19b6436ff15), C(0x139153392a3d3c0e), C(0xaa7073e62e1320c7), C(0xbf78b34efcef8d78), C(0x84489c6bf230039), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x707a475edc0daeb8), C(0x29f8aed3cdcee18c), C(0x3e1a048f2ac2411f), C(0xee68213edbb058c7), C(0x8a7b7e337ea21548), C(0x64095a066fcaf625), C(0xdb5da1fd34c007aa), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe9984cee385a5a35), C(0xbbdc22f9228537ff), C(0x9aeceefe4da32a83), C(0xf5e5d4162f3ec779), C(0x223e46bd893a0ae), C(0x19ea31f459d65113), C(0xa180e41d8318798d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6496e6fcee023508), C(0xb26ee80260c15488), C(0x5207b2e806c78692), C(0xa8202cce61225e6b), C(0x2d2cf89ca32359a8), C(0xba8a59f7d9008e05), C(0x69fc1c16d4be8c58), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb6d5e5bf8bb2f964), C(0x732235aade21519c), C(0x12a58d17f8546b47), C(0x5666396be37f4a69), C(0x98e5b57f2ac97936), C(0xeb8788b72696127f), C(0x71c2989a8a53ebb4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xdbb102307eed35fb), C(0xb3484a09d80aa994), C(0xcd100da053d56073), C(0xd19270dac980ced6), C(0x7b892e4650a7df0b), C(0x4e4d8950a28365b1), C(0x6f0cd7b6ba61d5e2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x323629d0bd9d465f), C(0x310dd105d03a8d5b), C(0x531bce95d8bb0eeb), C(0x6e0ddcb9cc6d3937), C(0x753d3d390c2d9a32), C(0xcfb08f1795529489), C(0x7eaf598850ceda3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe52af374f1d158be), C(0x1540d7337486830b), C(0x4e1a59660ad6e3f3), C(0x2191b4a5266f1740), C(0x41e3efa52d8dd39), C(0xc57887bb6ec91ed4), C(0x80d13a10713ec646), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x374c4ee36437b688), C(0x7d2ea1220d78ade0), C(0x4684b93b97de5ee6), C(0x65be14a8566e0643), C(0x6d444e179494fccb), C(0xd1141be601ee41b4), C(0xac76dc9245fc0837), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x13cec65c8d8c594c), C(0xbc9e42b6308f5df0), C(0x10726cad24db8336), C(0x10aa206327ab1e22), C(0xce43bf74dbdd04ef), C(0x9f82e94c161ea20c), C(0xe76c4bc30fcfa261), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xa6ef8d534eccf277), C(0xe88119e921bef254), C(0x2e0ec50e6fe979dc), C(0xf2ac5c3e12b9f171), C(0x2790e5110ce0524f), C(0x894b70e3c6e4afb3), C(0xa05105ad1319b8aa), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x821844d03da733d3), C(0xe557d6470b8f72e1), C(0x1deb4d3209b8d910), C(0x260d962023a628d7), C(0x184f8586b6fd7d81), C(0x25b98cb4875a4a67), C(0xd94535c207360a39), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x48566f12bee42c39), C(0x73656b26eea7e0c6), C(0x828ed07d0df800f2), C(0x3cf2b5f42d49d4ac), C(0x908e6cbb900b418), C(0x48cc84df6de8ccca), C(0xb993dbe71c0f6483), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb507ace18c7cca7b), C(0x2d061684043d6fd9), C(0xef1ef01d4ef75971), C(0x70d3bc91a638d3ec), C(0x6d68a1b462809c73), C(0x76f7eb2e29ee3a9d), C(0xb80d88b56a67ca36), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf1253a63d880e37), C(0x7f2e1644d1d3357c), C(0xc1965b600a8df257), C(0x99bdbcf20507d18a), C(0xeb9009144434ff3a), C(0x38d4e4846157a928), C(0x4aea39d3ae8cd0c1), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4a8340e68817da1e), C(0x137c6930cf9d0c04), C(0xd91cf5b059b6cd83), C(0x1c68d57e8bddb0f9), C(0xc38434f1d79fda98), C(0xbd754a99bcd8570c), C(0xb8ae64ca5bbcc99a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x59f6e03f588123ef), C(0x20fec99e9b33e2b0), C(0x85112fc684babffc), C(0x75e0ecb17e8a20a1), C(0xc9bb1eceaa6af6cb), C(0xb1b17b903b449aa0), C(0x91ba2c376af7ac61), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x76dca7203e5a5b5b), C(0xaa7adac414ebf447), C(0x731b56fa9fde24ae), C(0x346d3eebb7384335), C(0x242743d1a683240), C(0x324e97d0881c73d5), C(0xb9543da36e81325c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3a3beca1e183b2c7), C(0xe0699a97e3e8f816), C(0x84f22445fdc824d), C(0x6f45bcb999f800d3), C(0x534ce0b0a43511ef), C(0x254dd4f7fb14d078), C(0xe6140575c20d4632), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd0edbab00bb3b980), C(0x49202c3135bc6708), C(0xb44476275e7f5bd9), C(0x66648f4f22c62dc3), C(0x79ba17f561e2c0c3), C(0x35f877a56be052e9), C(0x37aaf15f89e71c97), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf2fcebfde8df2670), C(0x85e149139c7acb6), C(0x34a39ac3cef68ea7), C(0x3f8b01fd6ac57ca2), C(0x2b1f8573886d22c), C(0xdcf96c8ae5655697), C(0x240e3f7b77d260b3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x66e01c952e00b5a3), C(0x2c6772345fe7a505), C(0xce3c61ac41f0cf28), C(0xb380f6a78e765366), C(0xf17f6b377a788913), C(0x697fff4db9b53e14), C(0x43492d9c8b5f505c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x20e2597df93b1d82), C(0x4c72c0d07d634086), C(0x84e90170550c8c87), C(0xc50422963ab45fea), C(0xa0d6ef7e925e011e), C(0x1814011b139fceb1), C(0xfda187586fcde26c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x41e3d38b169f527), C(0x2db5c0fbfb5678b3), C(0xbb6240e6b7ae0db1), C(0x94c3658f08ecc6df), C(0x95945b99d1e01baa), C(0x61d90373b9976206), C(0x3f6ded31a41fe6e3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xcd413b22363ce08a), C(0xd03aab9932f8b25c), C(0xa85bc04a7044ae7e), C(0xcc4775e95f98e5b0), C(0x747a77374bda0ede), C(0x75599f9f86046db7), C(0x36ee979420c7fd55), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1a40632178de533b), C(0xa53d9a96304c96ae), C(0xb4f85409add3e762), C(0xb3bc1b4898b31850), C(0x45a5f9af894df43f), C(0x8d8d51189542e147), C(0xa6a30fc863d6cc9b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x333def8cc13e6e5c), C(0xaede2b5a56d901db), C(0xb0dc2f7bd8605a93), C(0x3facfd9eb3180cc2), C(0x3f9010d5547aca38), C(0xd9b64964a562025c), C(0xcca5a239a7c11eb5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x150a02c0af1d77f3), C(0xb9c0d3220c4bcb9e), C(0xccff413f0f13edd6), C(0xc4dde59bbf5e76ee), C(0x907e2c5e26fffc4c), C(0x6f8ce9bccf73980), C(0xb357053ef46b2411), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x95371497e074c817), C(0xa4151bbfb3341906), C(0xe5eef2a8eb18f5ee), C(0x67421b8a2b3454a3), C(0xf023fb99e6ff786a), C(0x8bbf1e63ae4555bb), C(0x1cd62fc126da4919), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb2bf9ed25d4c8309), C(0x19b620024f4a300f), C(0x1c6302f562942154), C(0xceba65bd88bf1d54), C(0xfbcc06f5fc6aa58), C(0xfef6b08a85f0c99f), C(0xf90d5788e574596f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x43391fb6d18c5538), C(0x7726591f70780aeb), C(0x1d3c6d82429ef36a), C(0x9932919799803aa5), C(0xcb1df42ecdb266c4), C(0xf83b3e9b3b4ec8d8), C(0x11320a5b32947007), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2e1ea34669165696), C(0xf279aba13156ee65), C(0x84df8da7a137b346), C(0xce472980208f4301), C(0x9636492ebc6fcece), C(0xdfa3c0b96ce21c53), C(0xb165b931de25ad56), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7f86624c2d4e6042), C(0x912c902f503e14f3), C(0x5a29bf6d021a8760), C(0xbad24f8b9e93f147), C(0xbde72955da1efb39), C(0xbd7de638c72bc20b), C(0x40dc53e2c1ae326), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x49682b3f2dbb2029), C(0x394cda9bd2137e30), C(0x1488f36480c8ae8e), C(0x427dbf4126eeb4bc), C(0xa6e6f536feed3543), C(0xc811a91f3c069063), C(0x2ae528cd45dae0da), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xf5120254c6f3254e), C(0x431c28f66c32c920), C(0xc352ef16416589e2), C(0xad2af2b85df04ebf), C(0xf1d9b1458e2c52d9), C(0xe13de9a1fbd1aa2), C(0x17ff3a50753968b0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x32322576dfa25afc), C(0xa0242f6aef829b4e), C(0xc7ebeef52ccbcb25), C(0x16d62a1715528f92), C(0x2d4db8f008ad36a7), C(0xf994e1e405664c82), C(0x4b26e6adb4a6b556), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x66093fe4d2cad02), C(0x21cbad2d7a679fc), C(0xfa8df1e26e9df524), C(0x3b2f255a374ef002), C(0x6e3d77bc6ff5b217), C(0x888b6f6e6a5a9945), C(0x7d3324c13fb3e7ba), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x710ae7b1967dd256), C(0x1b51927b9f4dca3), C(0x9c24f65fce7de3b8), C(0xacd81f0b50814169), C(0xe34bee74572aa8b5), C(0xfe341050acd3b0b0), C(0x4599e7919722d842), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbfa94295a75a74a9), C(0x428e96e3140ab83), C(0xebb411d3f9efaaa6), C(0xa4ca6e0ce09e138), C(0x504fb7336b9494d4), C(0xcb4253998a2e4c44), C(0xb16341c5e3853c02), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4c6bc11b3e283546), C(0xc183162862a2693c), C(0x378a84cd0a2c2bec), C(0xe1d5ec1998052c75), C(0x7d9ef7b1088f7234), C(0xb66554281b0a1d31), C(0x85973ab849e93af0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6b453210638ed039), C(0x6942a00e246bdfb1), C(0xf31dfc9986556060), C(0xb0f5e75947e76a6d), C(0x90c05b62642d3a24), C(0x930777028ebdfd58), C(0x25e83fb4e5bf341), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xfc6f39364fea6c32), C(0x585f983084da141d), C(0x3163e554aaf2c567), C(0x7d08a14d8d4c9317), C(0x42587a14e16f4fdb), C(0x47569062b450f6d7), C(0x98c4f6b7fe765461), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc6cde2e1487fa19d), C(0xe37fcf98fba7a664), C(0xe8d34907bb461ad), C(0x12a36abff6b29f29), C(0x3a5316a8502a35c2), C(0x7c93aaa35da06f5a), C(0xb32d0a314d7741ae), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc8e2e79ae7aeb0f1), C(0x10a274633805b953), C(0xe65c22a227bf96d7), C(0xd12dc763af6d81ea), C(0x17c9ad9f93280391), C(0x4f7c819724884dee), C(0x80f21ba5ffe98951), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x473b58475b95f2c3), C(0x49304f84cb5189d6), C(0x415bc1c0f72c1248), C(0x78c99f6b8046eebc), C(0x819b61aa6d979caf), C(0xe2bd7e31bce1bd8b), C(0x7e5f16df540ee87f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x23db4825404759e7), C(0x54b75cfed0eabd78), C(0xdda7dad48f449392), C(0xe2e68edd6ae8f1f7), C(0x8474c9e6a68ac6f5), C(0x186a8e05ac60cbc2), C(0xdd3b2f844b5a9ab3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x94a66d5551fa7da8), C(0x6271508fddd43aa0), C(0xcc43556efe0c881b), C(0x71d8ea7a4e2da192), C(0x5cc61c50808b92b0), C(0xb370ae85d547e21b), C(0xc7738839125270a2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xbfd6a834e74cba20), C(0x40c58c15b662ae35), C(0xbe483f266619c98d), C(0x4436f48df59b3103), C(0x4b59e29a30dc3b2e), C(0xe23c6f7a4ed2eec1), C(0x96598d8a6ec3d3a0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3b94c1621611b5d3), C(0x40218d960cebc668), C(0x88bb9de9f21d8439), C(0xb1d7cc8c9d5489e7), C(0xbb8073866bca352f), C(0xa9bdbd31ed2bc54f), C(0xea07ca18ff70af), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc2f345bf5ffea6a8), C(0xb323b0956324cca3), C(0x96c0d47cc60e6dbc), C(0xdb71521a37f2c1da), C(0x5a92d6e935d78fb7), C(0x6d212cafd181957a), C(0x95a3e096ca96c289), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2168600d492821a), C(0x3200f838d8d40893), C(0x2262f28374632720), C(0xa4b40a910cf02d0e), C(0xe0069718e4c0fb04), C(0x64418037c4dc60c0), C(0xb997faf4958c451c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9a4042e5e515051c), C(0xeba8e08b0eb2cf5e), C(0xe9d8adf2dc14126b), C(0x9ff9a47ac9506d80), C(0xdc52ddd79d5cc38e), C(0xe6c1ce8393e7e850), C(0xe114cd1ccecda312), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x45e534343b36b57d), C(0x1d66a3c528c0d9cd), C(0x41eb1cdf565297b6), C(0x537df3a47050fc25), C(0xfc197ef3283f78ee), C(0x6b893f5021b46292), C(0x95aca821e70a362b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc168e1c665ad91ac), C(0x114759de60c852f5), C(0x205c9226b8ac4edc), C(0xb7c3f2183f96752), C(0x8cc3ebc780253db1), C(0x460e4a4949028369), C(0x824baf2a2d8250b4), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd67661e9194d8215), C(0xda86d09543ad2d65), C(0x1cc96449839fd9d0), C(0x46818057af60d930), C(0x5991e521b7791255), C(0x10f85eb83e7f3513), C(0xf792a79849fd7f9a), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x755fde5bdfc44bfe), C(0xdb9ea1766c5f9c2e), C(0x718982318d3a1c64), C(0x847089cb841ff845), C(0x2307b8de874ed911), C(0xe12b3cf7cb7ddf01), C(0xc0f88460363c7c82), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5ea8f71a8db73a43), C(0x5db74aceab3217ad), C(0x571b6e5308de4096), C(0x70745db0fcab1747), C(0xd5760bf37e70a616), C(0x148b292497695a17), C(0x7f25e8fe3207381d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x469cdd38580cb319), C(0xce17f9268976005e), C(0x6fe0eb74927886b6), C(0x8c2c8f28aca17175), C(0x19c2f98273f000e6), C(0x5a3b34e0716f4702), C(0x349689e263bc91de), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x597c55972df98998), C(0x63d385818b67740e), C(0x800f53ebf8d939ee), C(0x9a467f399df6431f), C(0x8611b0947c4b03f2), C(0x32ab7af66286ba81), C(0xe2892f675cd6c5ad), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x8568c8479cb896f5), C(0x681dbd2a958dbd59), C(0xe949c6e65ff97daa), C(0x62b5dafed1fe308e), C(0x10e16cb6caa9d77a), C(0xcdd57f0442340fd3), C(0x4523f3ba5cef14d0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x1fc87401a28fff8a), C(0x7b36a64698569a26), C(0x4345702479af8cca), C(0xa1d036b397702b2d), C(0xd42a5a91d9d5a575), C(0x88ca9d88ba6ddd4c), C(0x5f71f09e5b451226), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x928460ee79e97cc9), C(0x8910f01505cee9e2), C(0x2d596829ea99f3ae), C(0xee49a43efc63f6dc), C(0xa5d3e38b71c4dc3a), C(0x39f4900072cebead), C(0xa751e82b791dddc2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x7e16adbb5d07ddbe), C(0x370ec6b51fe542a4), C(0xb1f18b985a06e694), C(0x25ae06f58136cdb7), C(0xf0e900e340023ac8), C(0x3d93c688274b90ce), C(0x8c43f98eb39744e7), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9b5d61b2b6d05a1c), C(0xcb1a5c97c3e83a04), C(0x38baf280e96fdad4), C(0x2615071835299ffd), C(0xdc3e6454981dd9f6), C(0x59c57d05d906916b), C(0xd217d52efd005b07), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xac364c5d44555598), C(0xb95b3aafe7c0f61), C(0x241928765076a8c5), C(0x15286f1de007cdcd), C(0xebdcf20a1e42822c), C(0x878c39e0c90cad81), C(0xf0f25302dbd3002), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x98c9764114d06cb), C(0xb929b5bd1f88ec7e), C(0xea0fdb316076c5f6), C(0x4a6d9608b0c0cd06), C(0xacd1afb5eaac7db5), C(0xff2be69a7769cb56), C(0xaba05bacfdd04f51), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xcb45892c5b62dccd), C(0x26244e8009a6757b), C(0xe10a86fc4738645e), C(0xa96a92a4a99e3d44), C(0x29150038e3a4da0b), C(0x379850aa8e09e651), C(0x64df693f3e228f16), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x84b4b8081de6c1d7), C(0xd5cca7fd2698e028), C(0x21dc6f2e60fa15f), C(0xdca769b8d56826e2), C(0xa999ab399268d1b1), C(0x70a703feba7c33c), C(0x53703405c1760a5), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9c80d0fb3c9a55d0), C(0x63fc3876a8f722a5), C(0xc108163a0da25f67), C(0x6fb56535b583ff1c), C(0x644dc8bc73364103), C(0x61a547684a3608ef), C(0xf3b120221dd9697d), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x540dabc6272e4b41), C(0x4c7508c2ba64b41d), C(0xeb52dd0e7e952501), C(0xf8d9a627dfd4d84d), C(0x4ea474e6bf766969), C(0xf916e2bcaeb83cf0), C(0x423f790d186051da), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x9b419d153385d259), C(0xb43c3bf5f33c12ee), C(0x5ccbcb21c6f085f3), C(0x99dc2fe7d318170), C(0x7a683261d7060fa8), C(0x350ad306bb7ccaac), C(0xbc3058d6e9921bcf), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xdd486d89bbb2a036), C(0x994c8cad09d1cc27), C(0xb86b6543fc6276ed), C(0x8cc69f3ac9ad58a6), C(0x6ef789683080f21f), C(0xb0a76477cc76407c), C(0xb5d634fd4ff0b9ce), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x6e02793fe8655e19), C(0xb49df281baf622da), C(0x5daf444b08f1724a), C(0x606fb7bd9de22458), C(0xc57c540c38311bf4), C(0xff533ef545b26999), C(0xc8ff449ebbd7da07), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb70edd368850e8cf), C(0xe4cb09362bf7cbf8), C(0xe1a21bd2de10df3c), C(0xbdc731127a12b873), C(0x4deb186e645edf8b), C(0x22c45afcfc3735ad), C(0x74ba9d334bb2e45c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xea6cf19f08e837d8), C(0xbfb7bc0773ca0972), C(0x7562496d26ba4e1c), C(0x309d751439f8cf4d), C(0x5d2583b4ed252c0), C(0xc12c6bf49806e465), C(0x71c6f349f7fb9236), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x4d197ae8f8c3523c), C(0x229dbe986014142d), C(0xc4dfa4c84d7f1125), C(0x5ec6683b7b2d1ee), C(0x2aed9f9cb5551531), C(0xd808e0a60d103428), C(0x162ba9dd2b749c63), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xddc514c5873048e1), C(0x24f6bbf3f0e63fde), C(0x29eac02af97de173), C(0xe628431493cb1598), C(0x7b8a889067a30d77), C(0xac25a0d190e03a14), C(0x6ade3e5cc83dfee7), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x49d4f98cb205cc9c), C(0xf3a3c7276ad923a4), C(0xf4a950ed9fe81ea9), C(0xe3c9c727815b69b7), C(0x2e62aaa96aea8969), C(0x69a2a38440a6d73c), C(0x1efcab6a109b6869), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x43c06116da2c6bd5), C(0x93e15b613f57f894), C(0x685ef99f855158aa), C(0xa69cd689d8f2724c), C(0x543186f1b59c7f0f), C(0xb818680983e7557), C(0x7659de18b274cbc8), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe5fe62e8b9a71c7a), C(0xb1f728ceda69d96f), C(0xfb916ec8ffd3c2b3), C(0x95f969dce1309381), C(0x18114e0511a57ce3), C(0x1ab2bf825a1d7e46), C(0x1eaabde6e5ff4962), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x164bb7f1d4e95803), C(0xc018bac02cf6023e), C(0x3001945c3463af44), C(0x8e5139fa1ef6b699), C(0x9b39a84475f0452), C(0xc4dc645e63193f61), C(0x402ff137e1021713), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xc4854498c31ab511), C(0xd3679668fa3978d4), C(0x5c9da1b987ca9d15), C(0xb4c8e8ba61b3ec04), C(0xfd894032e835fa18), C(0xd96f29fc7749c1a9), C(0x2c256fa1cc416ef7), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x2737d887d16334ee), C(0x39c454733b947f38), C(0xe8d56c31c29c82b0), C(0x4ab06a5c101ed75d), C(0xc265a403c3743bc2), C(0xe6bfa65c3681e7f5), C(0x44022a8bbe431c8e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x638135132965ee3b), C(0x58e998c0c0198517), C(0xb8427ac4983ff0f6), C(0x250c492e3fe6014a), C(0xe508f7260312b192), C(0xd922903ae658b136), C(0x6b837dcd9bc8371e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x22fa4df1fc7d7c1e), C(0x5d9eb9560a6830be), C(0xb6510889a4c6c199), C(0x48b36e190eb1e880), C(0x5b0981b3baca8559), C(0xb38d0a3946c6ba5a), C(0x288b25eaeba9c4c6), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x3b3363ce879dea68), C(0x7bc4a7bc73c61123), C(0xf4945426df80733a), C(0x3e9a8e4a49281c53), C(0x3159f8b713632101), C(0x9e1df1ae59800ff2), C(0xa6b20cc3eea56a8c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x5954b10428c23976), C(0xd58548f56e31e115), C(0x9fefeb60f97b3375), C(0x1c287cd84f8a50d1), C(0xd7ac67389fe5c511), C(0x3168f7bf076a315f), C(0xa9871b5f116c5d2c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x76b9a43060ef413e), C(0x25f326310feb2c7d), C(0xca96a690b8d0ea42), C(0x98a7e59d5ceb10a2), C(0xae4ec395375f4375), C(0x2e7265f625a6bc33), C(0xa51bc362dbf05fbb), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xae8372ddf645b30d), C(0x7356f94bb11f6549), C(0x75da652315d1a09c), C(0xe509a1f9480f34c), C(0x2e9fa5c3a84733ec), C(0x8063c65a7a1d4b90), C(0xf3554f5b50d443e3), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xd657c0473ec3119), C(0x99924c332a698786), C(0x17a95f45db8cac70), C(0x7fd31d283d81edc6), C(0xed8e4bb93b5157a3), C(0x26232db6f91c32d6), C(0xd4a772807fa51b3e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x72bb8dc446ce6a6a), C(0xc345f676420a2ac2), C(0xfa49460681f4ee11), C(0x2e6e0409152ebe85), C(0x73cb78ab751c370), C(0x8ff3d5c4440d0ce0), C(0x7ffedc68ebf9b662), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x68562a0f5145373a), C(0xbb476fe63e2b4f56), C(0xce59beee3118ad3c), C(0x3ec3955403c766dc), C(0x1f5ce88203477753), C(0x79bc267699ae2f7), C(0x88400bfa09690bb2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x37f7641651e95480), C(0x3fd5f20886756d1b), C(0x23ef7d5b02f7962b), C(0xc28da0cf67fbc584), C(0x13766e5700129c14), C(0x37402d371512a872), C(0xd6ea21778b77dcb2), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x57fd773d6afab1c2), C(0x4b859c43da103b95), C(0x9cced0296e31f1c7), C(0xb2c88e6b067decb4), C(0x39c6cf857fc1d673), C(0xfc3de39b07ea8bbf), C(0xfda8915492cb2f2b), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xb94e8a87235bcc93), C(0xcb9a8812c82f44f7), C(0x8760581f4501637e), C(0xc59cf7334dd9ecf4), C(0x8bb4d19143d61efc), C(0xceb885ff6a84154a), C(0x97674a15d034ea9f), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xdd611b418ed944af), C(0xc1b61857bbbddcb7), C(0x55da3f71b4f8de2), C(0xac548553d784b332), C(0xbcad57e9fd394e9), C(0xedb5dbf34cf563a), C(0x5c8b09dedbd2006), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x995d1a2e823ec116), C(0x202feba5224a0d9d), C(0xf1a57c9f687135e0), C(0xa93e11ef9795623c), C(0x3db7eb94f4bf9d2a), C(0x9cd41eff9189ea31), C(0xba884150d2b4aa21), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0xe86d78f841f3946f), C(0x901ddc4adbc5c875), C(0xd3605ac52698b68b), C(0x8308c55d9f34fd39), C(0x43fa8e525ff8c46f), C(0x811c48869b053a91), C(0x816a4f0731366a2e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x90aa9f8973cb1f60), C(0xf839f47021d2fb8b), C(0xc1b24260abc8913c), C(0x3038c5915608c16c), C(0x51c1f94d3d1cf33b), C(0x9aec5642526a9065), C(0xa2e0b9c0eeb73b3c), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, { C(0x201188a03a94bc94), C(0xcb76199fe78b5322), C(0xdb0e67ae3ab570e1), C(0x1e19a3882d1b408b), C(0x2353bd7982090d22), C(0xdb1a97e53cea4262), C(0x8c29ad1f6339693e), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0), C(0x0) }, }; } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/CityHashTests.cs ================================================ #region License // Copyright (c) 2011 Google, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. #endregion using System; using System.Buffers; using Octonica.ClickHouseClient.Protocol; using Xunit; namespace Octonica.ClickHouseClient.Tests { public partial class CityHashTests { const UInt64 k0 = 0xc3a5c85c97cb3127UL; const UInt64 kSeed0 = 1234567; const UInt64 kSeed1 = k0; static readonly UInt128 kSeed128 = new UInt128(kSeed1, kSeed0); const int kDataSize = 1 << 20; const int kTestSize = 300; byte[]? data; // Initialize data to pseudorandom values. void setup() { data = new byte[kDataSize]; UInt64 a = 9; UInt64 b = 777; for (int i = 0; i < kDataSize; i++) { a += b; b += a; a = (a ^ (a >> 41)) * k0; b = (b ^ (b >> 41)) * k0 + (UInt64) i; var u = (byte)(b >> 37); data[i] = u; //memcpy(data + i, &u, 1); // uint8 -> char } } //#define C(x) 0x ## x ## ULL private static UInt64 C(UInt64 v) { return v; } void Test(int index, int offset, int len) { var seq = new ReadOnlySequence(data, offset, len); UInt128 u = CityHash.CityHash128(seq); UInt128 v = CityHash.CityHash128WithSeed(seq, kSeed128); #if NET8_0_OR_GREATER Assert.Equal(new UInt128(testdata[index, 4], testdata[index, 3]), u); Assert.Equal(new UInt128(testdata[index, 6], testdata[index, 5]), v); #else Assert.Equal(testdata[index, 3], u.Low); Assert.Equal(testdata[index, 4], u.High); Assert.Equal(testdata[index, 5], v.Low); Assert.Equal(testdata[index, 6], v.High); #endif } [Fact] public void main() { setup(); int i = 0; for (; i < kTestSize - 1; i++) { Test(i, i * i, i); } Test(i, 0, kDataSize); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseColumnWriterTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Utils; using System; using System.Collections.Generic; using System.Data; using System.Globalization; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseColumnWriterTests : ClickHouseTestsBase, IClassFixture, IClassFixture { private const string TestTableName = "stream_insert_test"; private readonly TableFixture _tableFixture; public ClickHouseColumnWriterTests(TableFixture tableFixture) { _tableFixture = tableFixture; } [Fact] public async Task InsertValues() { await using var con = await OpenConnectionAsync(); var rangeStart = _tableFixture.ReserveRange(100); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { var columns = new object?[writer.FieldCount]; columns[writer.GetOrdinal("id")] = new[] { rangeStart, rangeStart + 1 }; columns[writer.GetOrdinal("num")] = new List {49999.99m, -999999.99999m}; await writer.WriteTableAsync(columns, 2, CancellationToken.None); } var cmd = con.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id>={rangeStart} AND id<{rangeStart + 2}"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetFieldValue(1, "NULL"); var num = reader.GetDecimal(2); Assert.Equal("NULL", str); switch (id - rangeStart) { case 0: Assert.Equal(49999.99m, num); break; case 1: Assert.Equal(-999999.99999m, num); break; default: Assert.False(true, $"Unexpected id: {id}"); break; } ++count; } } Assert.Equal(2, count); } [Fact] public async Task InsertValuesFromGeneratedColumns() { await using var con = await OpenConnectionAsync(); var rangeStart = _tableFixture.ReserveRange(10_000); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { var columns = new Dictionary { ["id"] = Enumerable.Range(rangeStart, int.MaxValue - rangeStart), ["str"] = Enumerable.Range(0, int.MaxValue).Select(i => i % 3 == 0 ? i % 5 == 0 ? "FizzBuzz" : "Fizz" : i % 5 == 0 ? "Buzz" : i.ToString()) }; await writer.WriteTableAsync(columns, 100, CancellationToken.None); } var cmd = con.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id>={rangeStart} AND id<{rangeStart + 10_000}"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); var num = reader.GetFieldValue(2, null); Assert.Null(num); var i = id - rangeStart; switch (str) { case "Fizz": Assert.True(i % 3 == 0); break; case "Buzz": Assert.True(i % 5 == 0); break; case "FizzBuzz": Assert.True(i % 3 == 0); Assert.True(i % 5 == 0); break; default: Assert.NotNull(str); Assert.True(int.TryParse(str, out var parsedStr)); Assert.Equal(i, parsedStr); break; } ++count; } } Assert.Equal(100, count); } [Fact] public async Task InsertValuesFromAsyncEnumerableColumn() { await using var con = await OpenConnectionAsync(); var rangeStart = _tableFixture.ReserveRange(10_000); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { var columns = new object?[writer.FieldCount]; columns[writer.GetOrdinal("id")] = Enumerable.Range(rangeStart, int.MaxValue - rangeStart); columns[writer.GetOrdinal("num")] = new AsyncTestFibSequence(); await writer.WriteTableAsync(columns, 63, CancellationToken.None); } var cmd = con.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id>={rangeStart} AND id<{rangeStart + 10_000} ORDER BY id"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { decimal? previous = null, current = null; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1, null); var num = reader.GetFieldValue(2, null); Assert.Null(str); var i = id - rangeStart; Assert.Equal(count, i); if (current == null) current = 1; else if (previous == null) previous = 1; else { var next = previous + current; previous = current; current = next; } Assert.Equal(current, num); ++count; } } Assert.Equal(63, count); } [Fact] public async Task InsertStringsWithEncoding() { await using var con = await OpenConnectionAsync(); var values = new List {"ноль", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять", "десять", "одиннадцать", "двенадцать", "тринадцать", "четырнадцать", "пятнадцать"}; var rangeStart = _tableFixture.ReserveRange(10_000); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { writer.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.UTF7)); var columns = new object?[writer.FieldCount]; columns[writer.GetOrdinal("id")] = Enumerable.Range(rangeStart, int.MaxValue - rangeStart); columns[writer.GetOrdinal("str")] = values; await writer.WriteTableAsync(columns, values.Count, CancellationToken.None); } var cmd = con.CreateCommand($"SELECT CAST(id - {rangeStart} AS Int32), convertCharset(str, 'UTF-7', 'cp1251'), num FROM {TestTableName} WHERE id>={rangeStart} AND id<{rangeStart + 10_000} ORDER BY id"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { reader.ConfigureColumn(1, new ClickHouseColumnSettings(Encoding.GetEncoding("windows-1251"))); while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1, null); var num = reader.GetFieldValue(2, null); Assert.Null(num); var originalValue = values[id]; Assert.Equal(originalValue, str); ++count; } } Assert.Equal(values.Count, count); } [Fact] public async Task InsertSingleRow() { await using var con = await OpenConnectionAsync(); var rangeStart = _tableFixture.ReserveRange(100); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName}(num, id, str) VALUES", CancellationToken.None)) { await writer.WriteRowAsync(new List {42m, rangeStart, "Hello"}, CancellationToken.None); writer.WriteRow(null, rangeStart + 1, "world!"); writer.WriteRow(new List { 42.5m, rangeStart + 2, DBNull.Value }); await writer.EndWriteAsync(CancellationToken.None); } var cmd = con.CreateCommand($"SELECT cast(T.id - {rangeStart} AS Int32) id, T.str, T.num FROM {TestTableName} AS T WHERE T.id>={rangeStart} AND T.id<{rangeStart + 100}"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetFieldValue(1, (string?) null); var num = reader.GetFieldValue(2, (decimal?) null); switch (id) { case 0: Assert.Equal("Hello", str); Assert.Equal(42m, num); break; case 1: Assert.Equal("world!", str); Assert.Null(num); break; case 2: Assert.Null(str); Assert.Equal(42.5m, num); break; default: Assert.True(id >= 0 && id <= 3, "Id is out of range."); break; } ++count; } } Assert.Equal(3, count); } [Fact] public async Task InsertArrayValues() { try { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_arr"); cmd.ExecuteNonQuery(); cmd = cn.CreateCommand($"CREATE TABLE {TestTableName}_arr(id Int32, arr Array(Nullable(String)), multi_arr Array(Array(Nullable(Decimal64(4))))) ENGINE=Memory"); cmd.ExecuteNonQuery(); var arr = new List> {new List {"foo", null, "bar"}, new List {null, null, null, null}, new List(0), new List {"1", "2", "Lorem ipsum"}}; var multiArr = new[] { new List(0), new List { new decimal?[] {1, 2, 3, null, 4, 5}, new decimal?[0], new decimal?[] {null, null, 6, null} }, new List { new decimal?[0] }, new List { new decimal?[0], new decimal?[0], new decimal?[0], new decimal?[] {7, 8, 9, 10} } }; await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {TestTableName}_arr VALUES", CancellationToken.None)) { var columns = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["ID"] = Enumerable.Range(0, int.MaxValue), ["ARR"] = arr, ["multi_ARR"] = multiArr }; await writer.WriteTableAsync(columns, Math.Max(arr.Count, multiArr.Length), CancellationToken.None); Assert.False(writer.IsClosed); writer.EndWrite(); Assert.True(writer.IsClosed); } cmd = cn.CreateCommand($"SELECT id, multi_arr, arr FROM {TestTableName}_arr"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var resultMultiArr = reader.GetFieldValue(1); var resultArr = reader.GetFieldValue(2); var expectedMultiArr = multiArr[id]; Assert.Equal(expectedMultiArr.Count, resultMultiArr.Length); for (int i = 0; i < expectedMultiArr.Count; i++) Assert.Equal(expectedMultiArr[i], resultMultiArr[i]); var expectedArr = arr[id]; Assert.Equal(expectedArr, resultArr); ++count; } } Assert.Equal(count, arr.Count); } finally { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_arr"); cmd.ExecuteNonQuery(); } } [Fact] public async Task InsertTupleValues() { var tuples = new[] { new Tuple>( 0, null, 4, 3434.35m, "0123456", null, 1, new Tuple(19, 3838838383, 53356.2343m)), new Tuple>( 1, "foo_bar", 534343434343434678, 8949679.5555m, "5432106", new DateTime(2020, 1, 1, 11, 11, 11), 255, new Tuple(-6598, 8984, 85760704949.4567m)), new Tuple>( 2, null, 0, 34987134.35m, "6543210", null, 128, new Tuple(-19, 38383, 0.2343m)), new Tuple>( 3, "one more value", 42, -0.6548m, "0246135", new DateTime(2019, 12, 31, 23, 59, 59), 15, new Tuple(null, null, -1234.567m)), new Tuple>( 4, null, ulong.MinValue, null, "1352460", null, 99, new Tuple(null, null, -0.0001m)), new Tuple>( 5, "Five!", ulong.MaxValue, 1_000_000.0001m, "2350146", null, 127, new Tuple(null, 2, -0.0042m)) }; try { await using var cn = new ClickHouseConnection(GetDefaultConnectionSettings()); cn.Open(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_tuple"); cmd.ExecuteNonQuery(); cmd = cn.CreateCommand($@"CREATE TABLE {TestTableName}_tuple(ten Tuple( Int32, Nullable(String), UInt64, Nullable(Decimal64(4)), FixedString(7), Nullable(DateTime), UInt8, Nullable(Int16), Nullable(UInt64), Decimal64(4) )) ENGINE = Memory"); cmd.ExecuteNonQuery(); await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {TestTableName}_tuple VALUES", CancellationToken.None)) { await writer.WriteTableAsync(new[] {tuples}, tuples.Length, CancellationToken.None); Assert.False(writer.IsClosed); await writer.EndWriteAsync(CancellationToken.None); Assert.True(writer.IsClosed); } cmd = cn.CreateCommand($"SELECT * FROM {TestTableName}_tuple"); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var value = reader.GetFieldValue>>(0); Assert.Equal(tuples[value.Item1], value); ++count; } } Assert.Equal(count, tuples.Length); } finally { await using var cn = new ClickHouseConnection(GetDefaultConnectionSettings()); cn.Open(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_tuple"); cmd.ExecuteNonQuery(); } } [Fact] public async Task InsertLowCardinalityValues() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_low_cardinality"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand($"CREATE TABLE {TestTableName}_low_cardinality(id Int32, str LowCardinality(Nullable(String)), strNotNull LowCardinality(String)) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); var idEnumerable = Enumerable.Range(0, 1000); var strEnumerable = Enumerable.Range(0, 1000).Select(NumToString); var strNotNullEnumerable = Enumerable.Range(0, 1000).Select(n => NumToString(n) ?? string.Empty); await using (var writer = connection.CreateColumnWriter($"INSERT INTO {TestTableName}_low_cardinality(id, str, strNotNull) VALUES")) { var source = new object[] {idEnumerable, strEnumerable, strNotNullEnumerable}; await writer.WriteTableAsync(source, 250, CancellationToken.None); await writer.WriteTableAsync(source, 250, CancellationToken.None); await writer.WriteTableAsync(source, 500, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } cmd.CommandText = $"SELECT id, str, strNotNull FROM {TestTableName}_low_cardinality"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1, null); var strNotNull = reader.GetString(2); var expectedStr = NumToString(id); Assert.Equal(expectedStr, str); Assert.Equal(expectedStr ?? string.Empty, strNotNull); ++count; } } Assert.Equal(1000, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_low_cardinality"); await cmd.ExecuteNonQueryAsync(); } static string? NumToString(int num) => num % 15 == 0 ? (num % 2 == 0 ? null : string.Empty) : num % 3 == 0 ? "foo" : num % 5 == 0 ? "bar" : num % 2 == 0 ? "true" : "false"; } [Fact] public async Task InsertEnumValues() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_enums"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand($"CREATE TABLE {TestTableName}_enums(id Int16, e8 Enum8('min' = -128, 'zero' = 0, 'max' = 127), e16 Enum16('unknown value' = 0, 'well known value' = 42, 'foo' = -1024, 'bar' = 108)) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); await using (var writer = connection.CreateColumnWriter($"INSERT INTO {TestTableName}_enums(id, e16, e8) VALUES")) { var source = new object[] { Enumerable.Range(0, 1000).Select(num => (short) num), Enumerable.Range(-500, 1000).Select(num => num % 108 == 0 ? "bar" : num < 0 ? "foo" : num == 42 ? "well known value" : "unknown value"), Enumerable.Range(0, 1000).Select(num => num % 3 == 0 ? sbyte.MinValue : num % 3 == 1 ? (sbyte) 0 : sbyte.MaxValue) }; await writer.WriteTableAsync(source, 1000, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } cmd.CommandText = $"SELECT e8, e16 FROM {TestTableName}_enums ORDER BY id"; int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); while (reader.Read()) { var e8 = reader.GetValue(0); var e16 = reader.GetInt16(1); Assert.Equal(count % 3 == 0 ? "min" : count % 3 == 1 ? "zero" : "max", e8); Assert.Equal((count - 500) % 108 == 0 ? 108 : count - 500 < 0 ? -1024 : count - 500 == 42 ? 42 : 0, e16); ++count; } Assert.Equal(1000, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_enums"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task InsertNullableEnumValues() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_null_enums"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand($"CREATE TABLE {TestTableName}_null_enums(id Int16, e8 Nullable(Enum8('min' = -128, 'zero' = 0, 'max' = 127)), e16 Nullable(Enum16('unknown value' = 0, 'well known value' = 42, 'foo' = -1024, 'bar' = 108))) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); await using (var writer = connection.CreateColumnWriter($"INSERT INTO {TestTableName}_null_enums(id, e16, e8) VALUES")) { var source = new object[] { Enumerable.Range(0, 1000).Select(num => (short) num), Enumerable.Range(-500, 1000).Select(num => num % 108 == 0 ? num % 5 == 0 ? null : "bar" : num < 0 ? "foo" : num == 42 ? "well known value" : "unknown value"), Enumerable.Range(0, 1000).Select(num => num % 3 == 0 ? num % 7 == 0 ? (sbyte?) null : sbyte.MinValue : num % 3 == 1 ? (sbyte) 0 : sbyte.MaxValue) }; await writer.WriteTableAsync(source, 1000, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } cmd.CommandText = $"SELECT e8, e16 FROM {TestTableName}_null_enums ORDER BY id"; int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); while (reader.Read()) { var e8 = reader.GetValue(0); var e16 = reader.GetFieldValue(1, (short?)null); Assert.Equal(count % 3 == 0 ? count % 7 == 0 ? (object)DBNull.Value : "min" : count % 3 == 1 ? "zero" : "max", e8); Assert.Equal((count - 500) % 108 == 0 ? (count - 500) % 5 == 0 ? (short?)null : 108 : count - 500 < 0 ? -1024 : count - 500 == 42 ? (short?)42 : 0, e16); ++count; } Assert.Equal(1000, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}_null_enums"); await cmd.ExecuteNonQueryAsync(); } } [Theory] [InlineData(0)] [InlineData(null)] [InlineData(2111)] [InlineData(999_990)] public async Task InsertLargeTable(int? maxBlockSize) { // "Large" means that the size of the table is greater than the size of the buffer const int rowCount = 100_000; var startId = _tableFixture.ReserveRange(rowCount); var settings = new ClickHouseConnectionStringBuilder(GetDefaultConnectionSettings()) {BufferSize = 4096}; await using var connection = new ClickHouseConnection(settings); await connection.OpenAsync(); await using (var writer = connection.CreateColumnWriter($"INSERT INTO {TestTableName}(id, str) VALUES")) { // Leave a default value intact when maxBlockSize == 0 if (maxBlockSize == null || maxBlockSize > 0) writer.MaxBlockSize = maxBlockSize; var table = new object[] {Enumerable.Range(startId, rowCount), Enumerable.Range(startId, rowCount).Select(num => num.ToString())}; await writer.WriteTableAsync(table, rowCount, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } await using var cmd = connection.CreateCommand($"SELECT id, str FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); await using var reader = cmd.ExecuteReader(); int expectedId = startId; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); Assert.Equal(expectedId, id); Assert.Equal(expectedId.ToString(), str); ++expectedId; } Assert.Equal(startId + rowCount, expectedId); } [Fact] public async Task InsertFromSecondaryInterfaces() { const int rowCount = 100; var startId = _tableFixture.ReserveRange(rowCount); // Each wrapper implements only one interface var ids = new EnumerableListWrapper(Enumerable.Range(startId, rowCount).ToList()); var strings = new GenericEnumerableListWrapper(Enumerable.Range(1, rowCount).Select(num => $"Str #{num}").ToList()); var numbers = new ListWrapper( Enumerable.Range(0, rowCount).Select(num => num % 17 == 0 ? (decimal?) null : Math.Round(num / (decimal) 17, 6)).ToList()); var connection = await OpenConnectionAsync(); var writeCount = (int) (rowCount * 0.9); await using (var writer = connection.CreateColumnWriter($"INSERT INTO {TestTableName}(id, str, num) VALUES")) { var table = new object[] {ids, strings, numbers}; await writer.WriteTableAsync(table, writeCount, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } await using var cmd = connection.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); await using var reader = cmd.ExecuteReader(); int count = 0; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); var num = reader.GetFieldValue(2, null); Assert.Equal(ids.List[count], id); Assert.Equal(strings.List[count], str); Assert.Equal(numbers.List[count], num); ++count; } Assert.Equal(writeCount, count); } [Fact] public async Task InsertValuesWithLowRowCount() { const int rowCount = 100, rowCap = 71; var startId = _tableFixture.ReserveRange(rowCount); await using var con = await OpenConnectionAsync(); var ids = Enumerable.Range(startId, rowCount).ToList(); var nums = Enumerable.Range(0, rowCount).Select(v => -100 + Math.Round(200m / (rowCount - 1) * v, 6)).ToArray(); await using (var writer = await con.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { var columns = new object?[writer.FieldCount]; columns[writer.GetOrdinal("id")] = ids; columns[writer.GetOrdinal("num")] = nums; await writer.WriteTableAsync(columns, rowCap, CancellationToken.None); } await using var cmd = con.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetFieldValue(1, "NULL"); var num = reader.GetDecimal(2); Assert.Equal(ids[count], id); Assert.Equal("NULL", str); Assert.Equal(nums[count], num); ++count; } } Assert.Equal(rowCap, count); } [Fact] public async Task InsertArrayFromMemory() { var ints = Enumerable.Range(0, 1000).ToArray(); var nums = ints.Select(v => -100 + Math.Round(200m / (ints.Length - 1) * v, 6)).ToArray(); var intsArr = new List>(); var numsArr = new List>(); int expectedCount = 0; for (int i = 0; i < ints.Length;) { var size = i % 13 == 0 ? 13 : i % 7; if (size == 0) size = 3; if (i + size > ints.Length) break; var intsMem = new Memory(ints, i, size); var numsMem = new Memory(nums, i, size); intsArr.Add(intsMem); numsArr.Add(numsMem); i += size; expectedCount = i; } await WithTemporaryTable("mem", "idx Int32, id Array(Int32), num Array(Decimal64(6))", RunTest); async Task RunTest(ClickHouseConnection connection, string tableName) { await using (var writer = connection.CreateColumnWriter($"INSERT INTO {tableName}(num, id, idx) VALUES")) { var source = new object[] { numsArr, intsArr, Enumerable.Range(0, numsArr.Count).ToArray() }; await writer.WriteTableAsync(source, numsArr.Count, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, num, idx FROM {tableName}"); int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); while (reader.Read()) { var idArr = reader.GetFieldValue(0); var numArr = reader.GetFieldValue(1); var idx = reader.GetInt32(2); Assert.Equal(idArr.Length, numArr.Length); for (int i = 0; i < idArr.Length; i++) { Assert.Equal(nums[idArr[i]], numArr[i]); Assert.Equal(intsArr[idx].Span[i], idArr[i]); Assert.Equal(numsArr[idx].Span[i], numArr[i]); } count += idArr.Length; } Assert.Equal(expectedCount, count); } } [Fact] public async Task InsertStringFromMemory() { const int rowCount = 400; var startId = _tableFixture.ReserveRange(rowCount); const string someText = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + "Donec varius tortor iaculis sapien malesuada, nec eleifend risus ultrices. " + "Suspendisse ac ligula nec nunc finibus lobortis sed ac ipsum. " + "Curabitur rutrum ligula feugiat, finibus enim id, vulputate purus. " + "Aliquam facilisis sem vel mattis fringilla. " + "Nullam in mauris feugiat, pulvinar nibh id, pretium quam. " + "Suspendisse hendrerit sapien et nisi rutrum, eu vestibulum magna convallis. " + "Nam sed turpis vulputate, volutpat augue eget, pulvinar sapien."; var ids = new List(Enumerable.Range(0, 100)); var mem = new List>(); var roMem = new List>(); var bytes = new List>(); var roBytes = new List>(); var someTextChars = someText.ToCharArray(); var someTextBytes = Encoding.ASCII.GetBytes(someText); Assert.Equal(someTextChars.Length, someTextBytes.Length); int position = 0; char[] separators = { ' ', ',', '.' }; for (int i = 0; i < ids.Count; i++) { while (separators.Contains(someText[position])) position = (position + 1) % someText.Length; var idx = someText.IndexOfAny(separators, position); var len = idx < 0 ? someText.Length - position : idx - position; mem.Add(new Memory(someTextChars, position, len)); roMem.Add(someText.AsMemory(position, len)); var bytesMem = new Memory(someTextBytes, position, len); bytes.Add(bytesMem); roBytes.Add(bytesMem); position = (position + len) % someText.Length; } var connection = await OpenConnectionAsync(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {TestTableName} VALUES", CancellationToken.None)) { var columns = new object?[writer.FieldCount]; columns[writer.GetOrdinal("id")] = ids.Select(id => id + startId); columns[writer.GetOrdinal("str")] = mem; await writer.WriteTableAsync(columns, ids.Count, CancellationToken.None); columns[writer.GetOrdinal("id")] = ids.Select(id => id + startId + ids.Count); columns[writer.GetOrdinal("str")] = roMem; await writer.WriteTableAsync(columns, ids.Count, CancellationToken.None); columns[writer.GetOrdinal("id")] = ids.Select(id => id + startId + ids.Count * 2); columns[writer.GetOrdinal("str")] = bytes; await writer.WriteTableAsync(columns, ids.Count, CancellationToken.None); columns[writer.GetOrdinal("id")] = ids.Select(id => id + startId + ids.Count * 3); columns[writer.GetOrdinal("str")] = roBytes; await writer.WriteTableAsync(columns, ids.Count, CancellationToken.None); } await using var cmd = connection.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); int count = 0; await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); var num = reader.GetFieldValue(2, (decimal?) null); Assert.Equal(mem[(id - startId) % ids.Count].ToString(), str); Assert.Null(num); ++count; } } Assert.Equal(rowCount, count); } [Fact] public async Task InsertFixedStringFromMemory() { const string someText = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + "Donec varius tortor iaculis sapien malesuada, nec eleifend risus ultrices. " + "Suspendisse ac ligula nec nunc finibus lobortis sed ac ipsum. " + "Curabitur rutrum ligula feugiat, finibus enim id, vulputate purus. " + "Aliquam facilisis sem vel mattis fringilla. " + "Nullam in mauris feugiat, pulvinar nibh id, pretium quam. " + "Suspendisse hendrerit sapien et nisi rutrum, eu vestibulum magna convallis. " + "Nam sed turpis vulputate, volutpat augue eget, pulvinar sapien."; var id = new List(Enumerable.Range(0, 100)); var mem = new List>(); var roMem = new List>(); var bytes = new List>(); var roBytes = new List>(); var someTextChars = someText.ToCharArray(); var someTextBytes = Encoding.ASCII.GetBytes(someText); Assert.Equal(someTextChars.Length, someTextBytes.Length); int position = 0; int maxLength = 0; char[] separators = {' ', ',', '.'}; for (int i = 0; i < id.Count; i++) { while (separators.Contains(someText[position])) position = (position + 1) % someText.Length; var idx = someText.IndexOfAny(separators, position); var len = idx < 0 ? someText.Length - position : idx - position; mem.Add(new Memory(someTextChars, position, len)); roMem.Add(someText.AsMemory(position, len)); var bytesMem = new Memory(someTextBytes, position, len); bytes.Add(bytesMem); roBytes.Add(bytesMem); maxLength = Math.Max(maxLength, len); position = (position + len) % someText.Length; } await WithTemporaryTable("fsm", $"id Int32, str FixedString({maxLength})", RunTest); async Task RunTest(ClickHouseConnection connection, string tableName) { await using (var writer = connection.CreateColumnWriter($"INSERT INTO {tableName}(id, str) VALUES")) { writer.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.ASCII)); await writer.WriteTableAsync(new object[] {id, mem}, id.Count, CancellationToken.None); await writer.WriteTableAsync(new object[] {id.Select(i => i + id.Count), roMem}, id.Count, CancellationToken.None); await writer.WriteTableAsync(new object[] {id.Select(i => i + id.Count*2), bytes}, id.Count, CancellationToken.None); await writer.WriteTableAsync(new object[] {id.Select(i => i + id.Count*3), roBytes}, id.Count, CancellationToken.None); await writer.EndWriteAsync(CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, str FROM {tableName}"); int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.ASCII)); while (reader.Read()) { var idVal = reader.GetFieldValue(0); var strVal = reader.GetFieldValue(1); Assert.Equal(roMem[idVal % id.Count].ToString(), strVal); count++; } Assert.Equal(id.Count * 4, count); } } [Fact] public async Task InsertFromCollectionOfObjects() { const int rowCount = 200; var startId = _tableFixture.ReserveRange(rowCount); // Each wrapper implements only one interface var ids = new EnumerableListWrapper(Enumerable.Range(startId, rowCount / 2).Cast().ToList()); var strings = new GenericEnumerableListWrapper(Enumerable.Range(1, rowCount / 2).Select(num => $"Str #1_{num}").ToList()); var numbers = new ListWrapper( Enumerable.Range(0, rowCount / 2).Select(num => num % 19 == 0 ? (object?)null : Math.Round(num / (decimal)19, 6)).ToList()); var ids2 = new AsyncEnumerableListWrapper(Enumerable.Range(startId + rowCount/2, rowCount / 2).Cast().ToList()); var strings2 = Enumerable.Range(1, rowCount / 2).Select(num => $"Str #2_{num}").ToArray(); var numbers2 = Enumerable.Range(rowCount / 2, rowCount / 2).Select(num => num % 19 == 0 ? (object?)null : Math.Round(num / (decimal)19, 6)).ToList(); await using var connection = await OpenConnectionAsync(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {TestTableName}(id, str, num) VALUES", CancellationToken.None)) { await writer.WriteTableAsync(new object[] { ids, strings, numbers }, rowCount / 2, CancellationToken.None); await writer.WriteTableAsync(new object[] { ids2, strings2, numbers2 }, rowCount / 2, CancellationToken.None); } await using var cmd = connection.CreateCommand($"SELECT id, str, num FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); await using var reader = cmd.ExecuteReader(); int count = 0; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); var num = reader.GetFieldValue(2, null); Assert.Equal(count + startId, id); if (id - startId < rowCount / 2) { Assert.Equal(strings.List[count], str); Assert.Equal(numbers.List[count], num); } else { Assert.Equal((string)strings2[count - rowCount / 2], str); Assert.Equal((decimal?)numbers2[count - rowCount / 2], num); } ++count; } Assert.Equal(rowCount, count); } [Fact] public async Task InsertValuesOfSpecifiedType() { const int rowCount = 200; var startId = _tableFixture.ReserveRange(rowCount); var ids = Enumerable.Range(startId, 123).ToList(); var list = new Int32ToUInt32MappedListWrapper(ids, v => (uint)v * 7); var table = new object?[] { list, list, null }; await using var connection = await OpenConnectionAsync(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {TestTableName}(id, num, str) VALUES", CancellationToken.None)) { Assert.Equal(typeof(int), writer.GetFieldType(0)); Assert.Equal(typeof(decimal), writer.GetFieldType(1)); Assert.Equal(typeof(string), writer.GetFieldType(2)); writer.ConfigureColumn("id", new ClickHouseColumnSettings(typeof(int))); writer.ConfigureColumn("num", new ClickHouseColumnSettings(typeof(uint))); Assert.Equal(typeof(int), writer.GetFieldType(0)); Assert.Equal(typeof(uint), writer.GetFieldType(1)); Assert.Equal(typeof(string), writer.GetFieldType(2)); await writer.WriteTableAsync(table, ids.Count, CancellationToken.None); } await using var cmd = connection.CreateCommand($"SELECT id, num FROM {TestTableName} WHERE id >= {{startId}} AND id < {{endId}} ORDER BY id"); cmd.Parameters.AddWithValue("startId", startId); cmd.Parameters.AddWithValue("endId", startId + rowCount); await using var reader = cmd.ExecuteReader(); int count = 0; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var num = reader.GetFieldValue(1); Assert.Equal(count + startId, id); Assert.Equal(id * 7, num); ++count; } Assert.Equal(ids.Count, count); } [Fact] public async Task InsertValuesFromObjectArrays() { var tableData = new object?[][] { new object[]{1, 2, 3, 4, 5, 6, 7}, new object?[]{"one", null, null, null, "five", "six", "seven"}, new object?[]{"192.168.121.0", DBNull.Value, "127.0.0.1", DBNull.Value, DBNull.Value, "10.0.0.1", "8.8.8.8"}, new object?[]{null, DBNull.Value, 12.34m, 12324.57m, 2195.99m, DBNull.Value, null}, new object[]{(sbyte)-1, (sbyte)0, (sbyte)0, (sbyte)1, (sbyte)1, (sbyte)-1, (sbyte)0} }; var expectedData = new object?[][] { tableData[0].Cast().Select(v=>(object)(long)v).ToArray(), tableData[1], new object?[]{IPAddress.Parse("192.168.121.0"), null, IPAddress.Parse("127.0.0.1"), null, null, IPAddress.Parse("10.0.0.1"), IPAddress.Parse("8.8.8.8")}, tableData[3], new object[]{"minus", "zero", "zero", "plus", "plus", "minus", "zero"} }; await WithTemporaryTable("obj_arr", "id Int64, str Nullable(String), ip Nullable(IPv4), num Nullable(Decimal32(2)), sign Enum8('minus'=-1, 'zero'=0, 'plus'=1)", RunTest); async Task RunTest(ClickHouseConnection connection, string tableName) { await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, str, ip, num, sign) VALUES", CancellationToken.None)) { writer.ConfigureColumn(0, new ClickHouseColumnSettings(tableData[0][0]!.GetType())); writer.ConfigureColumn(1, new ClickHouseColumnSettings(tableData[1][0]!.GetType())); writer.ConfigureColumn(2, new ClickHouseColumnSettings(tableData[2][0]!.GetType())); writer.ConfigureColumn(4, new ClickHouseColumnSettings(tableData[4][0]!.GetType())); await writer.WriteTableAsync(tableData, tableData[0].Length, CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, str, ip, num, sign FROM {tableName}"); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while (await reader.ReadAsync()) { for (int i = 0; i < tableData.Length; i++) { var expectedValue = expectedData[i][count]; bool isNull = expectedValue == null || expectedValue == DBNull.Value; Assert.Equal(isNull, reader.IsDBNull(i)); if (isNull) continue; var value = reader.GetValue(i); Assert.Equal(expectedValue, value); } ++count; } Assert.Equal(tableData[0].Length, count); } } [Fact] public async Task InsertMapValues() { var map1 = new Dictionary { ["a"] = 1, ["b"] = 2 }; var map2 = new KeyValuePair[] { new KeyValuePair("c", 3) }; var map3 = new List> { new Tuple("d", 5), new Tuple("e", 6) }; var map4 = new List<(string key, int value)> { ("f", 7), ("g", 8), ("h", 9), ("i", 10) }; await WithTemporaryTable("map", "id Int32, map Map(String, Int32)", Test); async Task Test(ClickHouseConnection cn, string tableName) { await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, map) VALUES", CancellationToken.None)) { await writer.WriteTableAsync(new object[] { AsListOfOne(1), AsListOfOne(map1) }, 1, CancellationToken.None); await writer.WriteTableAsync(new object[] { AsListOfOne(2), AsListOfOne(map2) }, 1, CancellationToken.None); await writer.WriteTableAsync(new object[] { AsListOfOne(3), AsListOfOne(map3) }, 1, CancellationToken.None); await writer.WriteTableAsync(new object[] { AsListOfOne(4), AsListOfOne(map4) }, 1, CancellationToken.None); } var cmd = cn.CreateCommand($"SELECT map FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while(await reader.ReadAsync()) { KeyValuePair[] expected; switch (++count) { case 1: expected = map1.ToArray(); break; case 2: expected = map2; break; case 3: expected = map3.Select(t => new KeyValuePair(t.Item1, t.Item2)).ToArray(); break; case 4: expected = map4.Select(t => new KeyValuePair(t.key, t.value)).ToArray(); break; default: Assert.True(false, "Too many rows."); throw new InvalidOperationException(); } var actual = reader.GetFieldValue[]>(0); Assert.Equal(expected, actual); } Assert.Equal(4, count); } static IReadOnlyList AsListOfOne(T value) { return new[] { value }; } } [Fact] public async Task InsertArrayLowCardinality() { var columns = new Dictionary() { ["id"] = Enumerable.Range(1, 10).ToList(), ["data"] = Enumerable.Range(1, 10).Select(o => new[] { $"test{ 1 + o * 2 % 3}", $"test{ 1 + (1 + o * 2) % 3}" }), }; await WithTemporaryTable("arrlc", "id Int32, data Array(LowCardinality(String))", Test); async Task Test(ClickHouseConnection cn, string tableName) { await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, data) VALUES", CancellationToken.None)) { await writer.WriteTableAsync(columns, 10, CancellationToken.None); } var cmd = cn.CreateCommand($"SELECT id, data FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while(await reader.ReadAsync()) { var id = reader.GetInt32(0); var data = reader.GetFieldValue(1); Assert.Equal(((List?)columns["id"])?[count], id); Assert.Equal(new[] { $"test{ 1 + (1 + count) * 2 % 3}", $"test{ 1 + (1 + (1 + count) * 2) % 3}" }, data); ++count; } Assert.Equal(10, count); } } [Fact] public Task InsertBoolValues() { var columns = new Dictionary { ["id"] = Enumerable.Range(1, 100).ToList(), ["b1"] = Enumerable.Range(1, 100).Select(i => i % 2 == 0).ToList(), ["b2"] = Enumerable.Range(1, 100).Select(i => i % 3 == 0 ? (bool?)null : i % 4 == 0).ToList(), ["b3"] = Enumerable.Range(1, 100).Select(i => (byte)(i % 8)).ToList(), ["b4"] = Enumerable.Range(1, 100).Select(i => i % 5 == 0 ? null : (byte?)(i % 16)) }; return WithTemporaryTable("bool", "id Int32, b1 Boolean, b2 Nullable(Boolean), b3 Boolean, b4 Nullable(Boolean)", Test); async Task Test(ClickHouseConnection cn, string tableName) { await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, b1, b2, b3, b4) VALUES", CancellationToken.None)) { await writer.WriteTableAsync(columns, 100, CancellationToken.None); } var cmd = cn.CreateCommand($"SELECT id, b1, b2, b3, b4 data FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); // Bool value can be true or false. // Byte value can be 1 or 0, even if inserted value was different. var b1 = reader.GetBoolean(1); var b1Byte = reader.GetByte(1); Assert.Equal(b1 ? 1 : 0, b1Byte); var b2 = reader.GetFieldValue(2, (bool?)null); var b2Byte = reader.GetFieldValue(2, (byte?)null); Assert.Equal(b2 == null ? null : b2.Value ? (byte?)1 : 0, b2Byte); var b3 = reader.GetValue(3); var b3Byte = reader.GetByte(3); var b3Bool = Assert.IsType(b3); Assert.Equal(b3Bool ? 1 : 0, b3Byte); var b4 = reader.GetValue(4); var b4Byte = reader.GetFieldValue(4, (byte?)null); var b4Bool = b4 == DBNull.Value ? (bool?)null : Assert.IsType(b4); Assert.Equal(b4Bool == null ? null : b4Bool.Value ? (byte?)1 : 0, b4Byte); Assert.Equal(count + 1, id); Assert.Equal(id % 2 == 0, b1); Assert.Equal(id % 3 == 0 ? (bool?)null : id % 4 == 0, b2); Assert.Equal(id % 8 != 0, b3); Assert.Equal(id % 5 == 0 ? DBNull.Value : (object)(id % 16 != 0), b4); ++count; } Assert.Equal(100, count); } } [Fact] public Task TransactionModeBlock() { return WithTemporaryTable("tran_block", "id Int32", Test); async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { var list = MappedReadOnlyList.Map(Enumerable.Range(0, 48).ToList(), i => i < 47 ? i : throw new IndexOutOfRangeException("Too long!")); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { // There will be three blocks in the list. The last block will produce an error, but first two blocks must be commited. writer.MaxBlockSize = 16; var ex = await Assert.ThrowsAnyAsync(() => writer.WriteTableAsync(new[] { list }, list.Count, ClickHouseTransactionMode.Block, ct)); Assert.NotNull(ex.InnerException); Assert.IsType(ex.InnerException); } var cmd = connection.CreateCommand($"SELECT * FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int expected = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); Assert.Equal(expected++, id); } Assert.Equal(32, expected); } } [Fact] public Task TransactionModeManual() { return WithTemporaryTable("tran_manual", "id Int32", Test); static async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteTableAsync(new[] { Enumerable.Range(0, 10) }, 10, ClickHouseTransactionMode.Manual, ct); await writer.RollbackAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(10, 10) }, 10, ClickHouseTransactionMode.Manual, ct); await writer.CommitAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(20, 10) }, 10, ClickHouseTransactionMode.Manual, ct); } await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteRowAsync(new object?[] { 18 }, false, ct); await writer.WriteRowAsync(new object?[] { 19 }, false, ct); await writer.RollbackAsync(ct); await writer.WriteRowAsync(new object?[] { 20 }, false, ct); await writer.WriteRowAsync(new object?[] { 21 }, true, ct); await writer.CommitAsync(ct); await writer.WriteRowAsync(new object?[] { 22 }, false, ct); await writer.WriteRowAsync(new object?[] { 23 }, false, ct); } var cmd = connection.CreateCommand($"SELECT * FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int expected = 10; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); Assert.Equal(expected++, id); } Assert.Equal(22, expected); } } [Theory] [InlineData(ClickHouseTransactionMode.Default)] [InlineData(ClickHouseTransactionMode.Auto)] public Task TransactionModeAuto(ClickHouseTransactionMode mode) { return WithTemporaryTable("tran_auto", "id Int32", Test); async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteTableAsync(new[] { Enumerable.Range(0, 10) }, 10, mode, ct); await writer.RollbackAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(10, 10) }, 10, mode, ct); await writer.CommitAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(20, 10) }, 10, mode, ct); } await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteRowAsync(new object?[] { 30 }, true, ct); await writer.RollbackAsync(ct); await writer.WriteRowAsync(new object?[] { 31 }, true, ct); await writer.CommitAsync(ct); await writer.WriteRowAsync(new object?[] { 32 }, true, ct); } var cmd = connection.CreateCommand($"SELECT * FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int expected = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); Assert.Equal(expected++, id); } Assert.Equal(33, expected); } } [Fact] public Task InsertMapLowCardinalityValues() { return WithTemporaryTable("map_lc", "id Int32, data Map(LowCardinality(String), Int32)", Test); static async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { var ids = Enumerable.Range(0, 1000).ToList(); var values = ids.Select( id => id % 2 == 0 ? new[] { KeyValuePair.Create("key1", id * 3), KeyValuePair.Create("key2", id * 3 + 1) } : new[] { KeyValuePair.Create("key2", id * 3 - 1) }) .ToList(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, data) VALUES", ct)) await writer.WriteTableAsync(new object[] { ids, values }, ids.Count, ct); var cmd = connection.CreateCommand($"SELECT id, data FROM {tableName} ORDER BY id"); await using (var reader = await cmd.ExecuteReaderAsync(ct)) { int counter = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); var data = reader.GetFieldValue[]>(1); Assert.Equal(counter++, id); Assert.Equal(values[id], data); } Assert.Equal(1000, counter); } // Skipping values await cmd.ExecuteNonQueryAsync(ct); } } [Fact] public Task TransactionModeAutoBackwardCompatibility() { // Check that the default transaction mode is 'Auto' when not specified return WithTemporaryTable("tran_auto_bc", "id Int32", Test); async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteTableAsync(new[] { Enumerable.Range(0, 10) }, 10, ct); await writer.RollbackAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(10, 10) }, 10, ct); await writer.CommitAsync(ct); await writer.WriteTableAsync(new[] { Enumerable.Range(20, 10) }, 10, ct); } await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id) VALUES", ct)) { await writer.WriteRowAsync(new object?[] { 30 }, ct); await writer.RollbackAsync(ct); await writer.WriteRowAsync(new object?[] { 31 }, ct); await writer.CommitAsync(ct); await writer.WriteRowAsync(new object?[] { 32 }, ct); } var cmd = connection.CreateCommand($"SELECT * FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int expected = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); Assert.Equal(expected++, id); } Assert.Equal(33, expected); } } [Fact] public Task InsertIPv6Values() { return WithTemporaryTable("ipv6", "id Int32, ip IPv6", Test, csb => csb.BufferSize = 33); static async Task Test(ClickHouseConnection connection, string tableName, CancellationToken ct) { var ids = Enumerable.Range(0, 255).ToList(); var ips = ids.Select(id => string.Format(CultureInfo.InvariantCulture, "192.168.0.{0}", id)).ToList(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, ip) VALUES", ct)) await writer.WriteTableAsync(new object[] { ids, ips }, ids.Count, ct); var cmd = connection.CreateCommand($"SELECT id, ip FROM {tableName}"); await using (var reader = await cmd.ExecuteReaderAsync(ct)) { int counter = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); var ip = reader.GetFieldValue(1); Assert.Equal(counter++, id); Assert.Equal(IPAddress.Parse("::ffff:" + ips[id]), ip); } Assert.Equal(255, counter); } } } [Fact] public Task InsertEmptyArrayLowCardinality() { return WithTemporaryTable("lc_array", "id Int32, strings Array(LowCardinality(String)), tuples Array(Array(Tuple(LowCardinality(String), LowCardinality(String))))", Test); static async Task Test(ClickHouseConnection cn, string tableName, CancellationToken ct) { var columns = new Dictionary { { "id", new[] { 123 } }, { "strings", new[] { Array.Empty() } }, { "tuples", new[] { new[] { Array.Empty<(string, string)>(), Array.Empty<(string, string)>() } } } }; await using (var writer = await cn.CreateColumnWriterAsync($"insert into {tableName} values", ct)) { await writer.WriteTableAsync(columns, 1, ct); } await using var reader = await cn.CreateCommand($"select * from {tableName}").ExecuteReaderAsync(ct); Assert.True(await reader.ReadAsync(ct)); var obj1 = reader.GetValue(0); var obj2 = reader.GetValue(1); var obj3 = reader.GetValue(2); Assert.Equal(123, obj1); Assert.Equal(Array.Empty(), obj2); var arr3 = Assert.IsType[][]>(obj3); Assert.Equal(2, arr3.Length); Assert.Equal(Array.Empty>(), arr3[0]); Assert.Equal(Array.Empty>(), arr3[1]); Assert.False(await reader.ReadAsync(ct)); } } [Fact] public Task InsertVariant() { return WithTemporaryTable("variant", "id Int32, v Variant(UInt64, LowCardinality(String), Array(Int32))", Test, afterOpen: (cn, ct) => cn.CreateCommand("SET allow_experimental_variant_type = 1").ExecuteNonQueryAsync(ct)); static async Task Test(ClickHouseConnection cn, string tableName, CancellationToken ct) { var columns = new Dictionary { ["id"] = new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }, ["v"] = new object?[] { DBNull.Value, "foo", "bar", Array.Empty(), "foo", null, "bar", new[] { 1, 2, 3, 4, 5 }, 42ul } }; await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {tableName} VALUES", ct)) await writer.WriteTableAsync(columns, 9, ct); var cmd = cn.CreateCommand(); cmd.CommandText = $"SELECT id, v FROM {tableName}"; await using var reader = await cmd.ExecuteReaderAsync(ct); var expected = (object?[])columns["v"]!; int count = 0; while(await reader.ReadAsync(ct)) { var id = reader.GetInt32(0); var val = reader.GetValue(1); bool isNull = (expected[id - 1] ?? DBNull.Value) == DBNull.Value; Assert.Equal(isNull, reader.IsDBNull(1)); Assert.Equal(expected[id - 1] ?? DBNull.Value, val); ++count; } Assert.Equal(9, count); } } protected override string GetTempTableName(string tableNameSuffix) { return $"{TestTableName}_{tableNameSuffix}"; } public class TableFixture : ClickHouseTestsBase, IDisposable { private int _identity; public TableFixture() { using var cn = OpenConnection(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}"); cmd.ExecuteNonQuery(); cmd = cn.CreateCommand($"CREATE TABLE {TestTableName}(id Int32, str Nullable(String), num Nullable(Decimal64(6))) ENGINE=Memory"); cmd.ExecuteNonQuery(); } /// /// Reserves a range of unique sequential identifiers /// /// The length of a range /// The first identifier of the reserved range public int ReserveRange(int length) { Assert.True(length > 0); var identity = _identity; while (true) { var nextIdentity = identity + length; var originalValue = Interlocked.CompareExchange(ref _identity, nextIdentity, identity); if (originalValue == identity) return identity; identity = originalValue; } } public void Dispose() { using var cn = OpenConnection(); var cmd = cn.CreateCommand($"DROP TABLE IF EXISTS {TestTableName}"); cmd.ExecuteNonQuery(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseCommandTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using System; using System.Collections.Generic; using System.Data; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseCommandTests : ClickHouseTestsBase { [Fact] public async Task SimpleExecuteScalar() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select 42"; var value = await cmd.ExecuteScalarAsync(CancellationToken.None); Assert.Equal((byte)42, value); } [Fact] public async Task ExecuteNonQuery() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select 42 as answer union all select 43 as not_an_answer"; var rowCount = await cmd.ExecuteNonQueryAsync(CancellationToken.None); Assert.Equal(2, rowCount); cmd.CommandText = "select 44"; var value = cmd.ExecuteScalar(); var byteValue = Assert.IsType(value); Assert.Equal(44, byteValue); } [Theory] [MemberData(nameof(ParameterModes))] public async Task Params(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select {a:UUID} a, {b} b, {long_parameter_name} /*+{_}*/ c, {d} d, {e123_456_789e} e, {_} f, {g} g, '{e123_456_789e}' h, '/v\\\\'`/v\\\\`, \"{a}\" i--{g} should not be replaced" + Environment.NewLine + "from (select 'some real value' as `{a}`)"; var now = DateTime.Now; now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, DateTimeKind.Local); var id = Guid.NewGuid(); cmd.Parameters.Add(new ClickHouseParameter("{a}") { DbType = DbType.String, Value = id.ToString("D") }); cmd.Parameters.Add(new ClickHouseParameter("{B}") { DbType = DbType.Guid, Value = DBNull.Value }); cmd.Parameters.Add(new ClickHouseParameter("LONG_parameter_NAME") { DbType = DbType.Int32 }); cmd.Parameters.Add(new ClickHouseParameter("D") { Value = 42m }); cmd.Parameters.Add(new ClickHouseParameter("{e123_456_789E}") { Value = "e123_456_789e" }); cmd.Parameters.Add(new ClickHouseParameter("_")); cmd.Parameters.Add(new ClickHouseParameter("g") { Value = now }); await using var reader = cmd.ExecuteReader(); Assert.True(await reader.ReadAsync()); Assert.Equal(id, reader.GetGuid(reader.GetOrdinal("a"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("b"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("c"))); Assert.Equal(42m, reader.GetDecimal(reader.GetOrdinal("d"))); Assert.Equal("e123_456_789e", reader.GetString(reader.GetOrdinal("e"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("f"))); Assert.Equal(now, reader.GetDateTime(reader.GetOrdinal("g"))); Assert.Equal("{e123_456_789e}", reader.GetString(reader.GetOrdinal("h"))); Assert.Equal("some real value", reader.GetString(reader.GetOrdinal("i"))); Assert.Equal(@"/v\", reader.GetString(reader.GetOrdinal(@"/v\"))); Assert.False(await reader.ReadAsync()); } [Theory] [MemberData(nameof(ParameterModes))] public async Task MsSqlLikeParams(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select cast(@a as UUID) a, @b b, @long_parameter_name /*+@_*/ c, @d d, {e123_456_789e} e, @_ f, {g} g, '@e123_456_789e' h, '/v\\\\\'`/v\\\\`, \"@a\" i--@g should not be replaced" + Environment.NewLine + "from (select 'some real value' as `@a`)"; var now = DateTime.Now; now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, DateTimeKind.Local); var id = Guid.NewGuid(); cmd.Parameters.Add(new ClickHouseParameter("{a}") { DbType = DbType.String, Value = id.ToString("D") }); cmd.Parameters.Add(new ClickHouseParameter("@B") { DbType = DbType.Guid, Value = DBNull.Value }); cmd.Parameters.Add(new ClickHouseParameter("@LONG_parameter_NAME") { DbType = DbType.Int32 }); cmd.Parameters.Add(new ClickHouseParameter("@D") { Value = 42m }); cmd.Parameters.Add(new ClickHouseParameter("e123_456_789E") { Value = "e123_456_789e" }); cmd.Parameters.Add(new ClickHouseParameter("@_")); cmd.Parameters.Add(new ClickHouseParameter("g") { Value = now }); await using var reader = cmd.ExecuteReader(); Assert.True(await reader.ReadAsync()); Assert.Equal(id, reader.GetGuid(reader.GetOrdinal("a"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("b"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("c"))); Assert.Equal(42m, reader.GetDecimal(reader.GetOrdinal("d"))); Assert.Equal("e123_456_789e", reader.GetString(reader.GetOrdinal("e"))); Assert.True(reader.IsDBNull(reader.GetOrdinal("f"))); Assert.Equal(now, reader.GetDateTime(reader.GetOrdinal("g"))); Assert.Equal("@e123_456_789e", reader.GetString(reader.GetOrdinal("h"))); Assert.Equal("some real value", reader.GetString(reader.GetOrdinal("i"))); Assert.Equal(@"/v\", reader.GetString(reader.GetOrdinal(@"/v\"))); Assert.False(await reader.ReadAsync()); } [Theory] [MemberData(nameof(ParameterModes))] public async Task EmptyStringParam(ClickHouseParameterMode parameterMode) { const string query = @"select {url}, {data}"; await using var cn = await OpenConnectionAsync(parameterMode); await using var cmd = cn.CreateCommand(query); cmd.Parameters.AddWithValue("url", ""); cmd.Parameters.AddWithValue("data", "{\"value\":107}"); await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); Assert.Equal((string)cmd.Parameters["url"].Value!, reader.GetString(0)); Assert.Equal((string)cmd.Parameters["data"].Value!, reader.GetString(1)); Assert.False(await reader.ReadAsync()); } [Theory] [MemberData(nameof(ParameterModes))] public async Task TypedParams(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); await using var cmd = cn.CreateCommand(); cmd.CommandText = "SELECT {param:UInt32} intVal, {PaRaM:Decimal128(9)} decVal, {pArAm:String} strVal"; cmd.Parameters.Add(new ClickHouseParameter("{Param}") { Value = "42" }); await using var reader = cmd.ExecuteReader(); Assert.True(await reader.ReadAsync()); Assert.Equal(42, reader.GetFieldValue(0)); Assert.Equal(42, reader.GetDecimal(1)); Assert.Equal("42", reader.GetString(2)); Assert.False(await reader.ReadAsync()); } [Fact] public async Task UseConnectionInParallel() { await using var cn = await OpenConnectionAsync(); var res = await Task.WhenAll(Sleep(cn, 1), Sleep(cn, 2), Sleep(cn, 3)); Assert.Equal(new[] { 1, 2, 3 }, res); async Task Sleep(ClickHouseConnection connection, int value) { await using var cmd = connection.CreateCommand(string.Format(CultureInfo.InvariantCulture, "SELECT sleep(0.2)+{0}", value)); return await cmd.ExecuteScalarAsync(); } } [Fact] public async Task ReadScalarFromLargeResultSet() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT number+42 FROM system.numbers"); var cts = new CancellationTokenSource(); var expectedTask = cmd.ExecuteScalarAsync(cts.Token); var task = await Task.WhenAny(expectedTask, Task.Delay(2000, CancellationToken.None)); if (!ReferenceEquals(expectedTask, task)) cts.Cancel(true); var value = await expectedTask; Assert.Equal(42, value); } [Fact] public async Task UseCommandInParallel() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT sleep(0.2) + 42"); var result = await Task.WhenAll(cmd.ExecuteScalarAsync(), cmd.ExecuteScalarAsync(), cmd.ExecuteScalarAsync()); Assert.Equal(new uint[] { 42, 42, 42 }, result); } #if DEBUG [Fact] public async Task CancelOnSocketTimeout() { // Attempt to emulate a network error. It doesn't work sometimes because ClickHouse server sends Progress messages var csb = new ClickHouseConnectionStringBuilder(GetDefaultConnectionSettings()) { ReadWriteTimeout = 25 }; await using (var cn = new ClickHouseConnection(csb)) { await cn.OpenAsync(); await using var cmd = cn.CreateCommand("SELECT sleep(3)"); var ioEx = Assert.Throws(() => cmd.ExecuteNonQuery()); var socketEx = Assert.IsType(ioEx.InnerException); Assert.Equal(SocketError.TimedOut, socketEx.SocketErrorCode); } } #endif [Fact] public async Task CancelOnCommandTimeout() { var csb = new ClickHouseConnectionStringBuilder(GetDefaultConnectionSettings()) { CommandTimeout = 2 }; await using (var cn = new ClickHouseConnection(csb)) { await cn.OpenAsync(); await using var cmd = cn.CreateCommand("SELECT sleep(3)"); Assert.Throws(() => cmd.ExecuteNonQuery()); } } [Fact] public async Task CancelOnTokenTimeout() { await using (var cn = await OpenConnectionAsync()) { await using var cmd = cn.CreateCommand("SELECT sleep(3)"); var tokenSource = new CancellationTokenSource(); tokenSource.CancelAfter(500); var ex = await Assert.ThrowsAnyAsync(() => cmd.ExecuteNonQueryAsync(tokenSource.Token)); Assert.Equal(tokenSource.Token, ex.CancellationToken); } } [Fact] public void ShouldUnwrapCommandExecuteScalar() { using var conn = OpenConnection(); Assert.Throws(() => conn.CreateCommand("select nothing").ExecuteScalar()); } [Fact] public async Task ExecuteScalarShouldReturnSingleResult() { await using var connection = await OpenConnectionAsync(); await using var cmdDrop = connection.CreateCommand("select 2+2"); var result = Convert.ToInt32(await cmdDrop.ExecuteScalarAsync()); Assert.Equal(4, result); } [Fact] public async Task CommandBehaviorCloseConnection() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT * FROM system.numbers LIMIT 100"); await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection)) { Assert.True(await reader.ReadAsync()); Assert.Equal(ConnectionState.Open, cn.State); } Assert.Equal(ConnectionState.Closed, cn.State); await cn.OpenAsync(); await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection)) { Assert.True(await reader.ReadAsync()); Assert.Equal(ConnectionState.Open, cn.State); await reader.CloseAsync(); Assert.Equal(ConnectionState.Closed, cn.State); } await cn.OpenAsync(); await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection)) { int count = 0; while (await reader.ReadAsync()) { Assert.Equal(ConnectionState.Open, cn.State); ++count; } Assert.Equal(100, count); Assert.Equal(ConnectionState.Closed, cn.State); Assert.False(await reader.ReadAsync()); } cmd.CommandText = "select x, sum(y) as v from (SELECT number%3 + 1 as x, number as y FROM numbers(10)) group by x with totals;"; await cn.OpenAsync(); await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection)) { int count = 0; while (await reader.ReadAsync()) { Assert.Equal(ConnectionState.Open, cn.State); ++count; } Assert.Equal(3, count); Assert.Equal(ConnectionState.Open, cn.State); Assert.True(await reader.NextResultAsync()); Assert.True(await reader.ReadAsync()); Assert.Equal(ConnectionState.Open, cn.State); Assert.False(await reader.ReadAsync()); Assert.Equal(ConnectionState.Closed, cn.State); Assert.False(await reader.ReadAsync()); Assert.False(await reader.NextResultAsync()); } await cn.OpenAsync(); cmd.CommandText = "It IS NOT a query..."; await Assert.ThrowsAsync(() => cmd.ExecuteReaderAsync(CommandBehavior.CloseConnection)); Assert.Equal(ConnectionState.Closed, cn.State); } [Fact] public async Task TableParameterSingleColumn() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT toInt32(number) FROM numbers(100000) WHERE number IN param_table"); var tableProvider = new ClickHouseTableProvider("param_table", 100); tableProvider.AddColumn(Enumerable.Range(500, int.MaxValue / 2)); cmd.TableProviders.Add(tableProvider); var expectedValues = new HashSet(Enumerable.Range(500, 100)); await using var reader = await cmd.ExecuteReaderAsync(); while(await reader.ReadAsync()) { var value = reader.GetInt32(0); Assert.True(expectedValues.Remove(value)); } Assert.Empty(expectedValues); } [Theory] [MemberData(nameof(ParameterModes))] public async Task TableParameterAndScalarParameter(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT toInt32(number) FROM numbers(100000) WHERE number IN (SELECT {val}*val FROM param_table)"); var tableProvider = new ClickHouseTableProvider("param_table", 100); tableProvider.AddColumn("val", Enumerable.Range(500, int.MaxValue / 2)); cmd.TableProviders.Add(tableProvider); cmd.Parameters.AddWithValue("val", 3); var expectedValues = new HashSet(Enumerable.Range(500, 100).Select(v => v * 3)); await using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { var value = reader.GetInt32(0); Assert.True(expectedValues.Remove(value)); } Assert.Empty(expectedValues); } [Theory] [MemberData(nameof(ParameterModes))] public async Task MultipleTableParameters(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT T2.id AS id, T1.id AS user_id, T1.value AS user, T2.value AS ip FROM q_user AS T1 LEFT JOIN q_addr AS T2 ON T1.id = T2.user_id WHERE T2.id%{param} != 0 ORDER BY T2.id"); var users = new (int id, string name)[] { (1, "user1"), (2, "user2"), (3, "admin1"), (4, "admin2") }; var addr = new (int id, string? ip)[] { (1, "8.8.8.8"), (2, "9.9.9.9"), (3, null), (4, null), (3, "127.0.0.1"), (4, "127.0.0.1"), (1, "2001:0db8:0000:0000:0000:ff00:0042:8329"), (4, "::ffff:192.0.2.1"), (2, "fe80::883b:771b:71c6:7c31") }; var usersTable = new ClickHouseTableProvider("q_user", users.Length); usersTable.AddColumn("id", users.Select(u => u.id)); usersTable.AddColumn("value", users.Select(u => u.name)); cmd.TableProviders.Add(usersTable); var addrTable = new ClickHouseTableProvider("q_addr", addr.Length); addrTable.AddColumn("id", Enumerable.Range(0, addr.Length)); addrTable.AddColumn("user_id", addr.Select(a => a.id)); var ipColumn = addrTable.AddColumn("value", addr.Select(a => a.ip)); ipColumn.ClickHouseDbType = ClickHouseDbType.IpV6; ipColumn.IsNullable = true; cmd.TableProviders.Add(addrTable); cmd.Parameters.AddWithValue("param", 7); await using var reader = await cmd.ExecuteReaderAsync(); int expectedId = 1; while(await reader.ReadAsync()) { var id = reader.GetInt32(0); Assert.Equal(expectedId, id); var userId = reader.GetInt32(1); var user = reader.GetString(2); var ip = reader.GetFieldValue(3, null); Assert.Equal(addr[expectedId].id, userId); var expectedUser = users.Single(u => u.id == userId).name; Assert.Equal(expectedUser, user); var expectedIp = addr[expectedId].ip == null ? null : IPAddress.Parse(addr[expectedId].ip).MapToIPv6(); Assert.Equal(expectedIp, ip); if (++expectedId % 7 == 0) ++expectedId; } Assert.Equal(addr.Length, expectedId); } [Fact] public async Task InsertWithParameters() { try { await using var connection = await OpenConnectionAsync(builder => builder.ParametersMode = ClickHouseParameterMode.Interpolate); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS insert_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE insert_with_parameters_test(str_val String) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO insert_with_parameters_test(str_val) VALUES (@str_param)"; var p = cmd.CreateParameter(); p.ParameterName = "str_param"; var insertedValue = "IZyy8d\\'\"\n\t\v\b\rLsVeTtdfk6MjJl"; p.Value = insertedValue; cmd.Parameters.Add(p); await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.CommandText = "SELECT str_val FROM insert_with_parameters_test"; var selectedValue = (string?)await cmd.ExecuteScalarAsync(); Assert.Equal(insertedValue, selectedValue); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS insert_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task DeleteWithParameters() { try { await using var connection = await OpenConnectionAsync(builder => builder.ParametersMode = ClickHouseParameterMode.Interpolate); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS delete_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE delete_with_parameters_test(str_val String) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "alter table delete_with_parameters_test delete where str_val = @str_param"; var p = cmd.CreateParameter(); p.ParameterName = "str_param"; var insertedValue = "IZyy8d\\'\"\n\t\v\b\rLsVeTtdfk6MjJl"; p.Value = insertedValue; cmd.Parameters.Add(p); await cmd.ExecuteNonQueryAsync(); // Assert: Not Throws } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS delete_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task UpdateWithParameters() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS update_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE update_with_parameters_test(str_val String) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "alter table update_with_parameters_test update str_val = @str_param where str_val = @str_param"; var p = cmd.CreateParameter(); p.ParameterName = "str_param"; var insertedValue = "IZyy8d\\'\"\n\t\v\b\rLsVeTtdfk6MjJl"; p.Value = insertedValue; ((ClickHouseParameter)p).ParameterMode = ClickHouseParameterMode.Interpolate; cmd.Parameters.Add(p); await cmd.ExecuteNonQueryAsync(); // Assert: Not Throws } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS update_with_parameters_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task SelectWithOffsetLimitParameters() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("select cast(42 as UInt64) limit @Limit offset @Offset"); cmd.ParametersMode = ClickHouseParameterMode.Interpolate; var p_limit = cmd.CreateParameter(); var p_offset = cmd.CreateParameter(); p_limit.ParameterName = "Limit"; p_limit.Value = 1; p_offset.ParameterName = "Offset"; p_offset.Value = 0; cmd.Parameters.Add(p_limit); cmd.Parameters.Add(p_offset); var result = await cmd.ExecuteScalarAsync(); Assert.IsType(result); Assert.Equal(42UL, result); } [Fact] public async Task CreateTableWithCommentParameter() { var tableName = GetTempTableName("with_comment"); try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {tableName}"); await cmd.ExecuteNonQueryAsync(); const string commentText = "The comment: \\'\"\n\t\v\b\rals;kdjang"; cmd = connection.CreateCommand($"CREATE TABLE {tableName}(id Int32) ENGINE = Memory COMMENT {{comment}}"); cmd.Parameters.AddWithValue("comment", commentText).ParameterMode = ClickHouseParameterMode.Interpolate; await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("SELECT comment FROM system.tables WHERE name = {table}"); cmd.Parameters.AddWithValue("table", tableName).ParameterMode = ClickHouseParameterMode.Serialize; var result = await cmd.ExecuteScalarAsync(); Assert.IsType(result); Assert.Equal(commentText, result); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {tableName}"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task WithQueryId() { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT * FROM system.processes WHERE query_id = {qid}"; cmd.QueryId = "ABSOLUTELY NOT a UUID"; cmd.Parameters.AddWithValue("qid", cmd.QueryId); await using var reader = cmd.ExecuteReader(); Assert.True(await reader.ReadAsync()); var isInitialIdx = reader.GetOrdinal("is_initial_query"); Assert.True(isInitialIdx >= 0); var queryIdIdx = reader.GetOrdinal("query_id"); Assert.True(queryIdIdx >= 0); var initialQueryIdIdx = reader.GetOrdinal("initial_query_id"); Assert.True(initialQueryIdIdx >= 0); bool isInitialQuery = reader.GetBoolean(isInitialIdx); Assert.True(isInitialQuery); var queryId = reader.GetString(queryIdIdx); var initialQueryId = reader.GetString(initialQueryIdIdx); Assert.Equal(queryId, initialQueryId); Assert.Equal(queryId, cmd.QueryId); Assert.False(await reader.ReadAsync()); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseConnectionStringBuilderTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseConnectionStringBuilderTests { [Fact] public void FromString() { var builder = new ClickHouseConnectionStringBuilder( "host=ClickHouse.example.com;port=65500;compress = off;DataBase=\"don't; connect\"\" to me :)\";user=root; password=123456;\r\n bufferSize=1337; ReadWriteTimeout=42; clientname=ClickHouse.NetCore Tests;clientversion=3.2.1"); var settings = builder.BuildSettings(); Assert.Equal("ClickHouse.example.com", settings.Host); Assert.Equal(65500, settings.Port); Assert.Equal("don't; connect\" to me :)", settings.Database); Assert.Equal("root", settings.User); Assert.Equal("123456", settings.Password); Assert.Equal(1337, settings.BufferSize); Assert.Equal(42, settings.ReadWriteTimeout); Assert.Equal("ClickHouse.NetCore Tests", settings.ClientName); Assert.Equal(new ClickHouseVersion(3, 2, 1), settings.ClientVersion); Assert.Equal(false, settings.Compress); } [Fact] public void Remove() { var builder = new ClickHouseConnectionStringBuilder {Host = "some_instance.example.com", Port = ClickHouseConnectionStringBuilder.DefaultPort *2, ClientName = "Test!"}; Assert.Equal(3, builder.Count); Assert.True(builder.Remove("pORT")); Assert.Equal(2, builder.Count); Assert.Equal(ClickHouseConnectionStringBuilder.DefaultPort, builder.Port); Assert.False(builder.Remove("Port")); Assert.True(builder.Remove("ClientName")); Assert.Equal(1, builder.Count); Assert.Equal(ClickHouseConnectionStringBuilder.DefaultClientName, builder.ClientName); } [Fact] public void SetConnectionString() { var builder = new ClickHouseConnectionStringBuilder( "host=ClickHouse.example.com;port=65500;DataBase=\"don't; connect\"\" to me :)\";user=root; password=123456;\r\n bufferSize=1337; ReadWriteTimeout=42; clientname=ClickHouse.NetCore Tests;clientversion=3.2.1"); Assert.Equal(9, builder.Count); builder.ConnectionString = "host=localhost; port=31337"; Assert.Equal(2, builder.Count); } [Fact] public void Default() { var builder = new ClickHouseConnectionStringBuilder(); Assert.Equal(string.Empty, builder.ConnectionString); Assert.Empty(builder); // The host is required Assert.Throws(() => builder.BuildSettings()); builder.Host = "localhost"; Assert.Equal("Host=localhost", builder.ConnectionString); var settings = builder.BuildSettings(); var checkedPropertiesCount = 0; Assert.Equal("localhost", settings.Host); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultPort, settings.Port); ++checkedPropertiesCount; Assert.Null(settings.Database); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultUser, settings.User); ++checkedPropertiesCount; Assert.Null(settings.Password); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultBufferSize, settings.BufferSize); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultReadWriteTimeout, settings.ReadWriteTimeout); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultClientName, settings.ClientName); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultClientVersion, settings.ClientVersion); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultCompress, settings.Compress); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultCommandTimeout, settings.CommandTimeout); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultTlsMode, settings.TlsMode); ++checkedPropertiesCount; Assert.Null(settings.RootCertificate); ++checkedPropertiesCount; Assert.True(settings.ServerCertificateHash.IsEmpty); ++checkedPropertiesCount; Assert.Equal(ClickHouseConnectionStringBuilder.DefaultParametersMode, settings.ParametersMode); ++checkedPropertiesCount; Assert.Null(settings.QuotaKey); ++checkedPropertiesCount; Assert.Equal(checkedPropertiesCount, settings.GetType().GetProperties().Length); } [Fact] public void Clone() { var builder = new ClickHouseConnectionStringBuilder( "host=ClickHouse.example.com;" + "port=65500;" + "DataBase=\"don't; connect\";" + "user=root; " + "password=123456;\r\n " + "bufferSize=1337; " + "ReadWriteTimeout=42; " + "clientname=ClickHouse.NetCore Tests;" + "clientversion=3.2.1;" + $"COMPRESS={!ClickHouseConnectionStringBuilder.DefaultCompress};" + "CommandTimeout=123;" + "TLSMode=rEqUIrE;" + "RootCertificate=/usr/local/share/ca-certificates/Yandex/YandexInternalRootCA.pem;" + "ServerCertificateHash=1234-5678 9abc-def0;" + "ParametersMode=Interpolate;" + "QuotaKey='unlimited'"); var settings = builder.BuildSettings(); var builderCopy = new ClickHouseConnectionStringBuilder(settings); settings = builderCopy.BuildSettings(); var checkedPropertiesCount = 0; Assert.Equal("ClickHouse.example.com", settings.Host); ++checkedPropertiesCount; Assert.Equal(65500, settings.Port); ++checkedPropertiesCount; Assert.Equal("don't; connect", settings.Database); ++checkedPropertiesCount; Assert.Equal("root", settings.User); ++checkedPropertiesCount; Assert.Equal("123456", settings.Password); ++checkedPropertiesCount; Assert.Equal(1337, settings.BufferSize); ++checkedPropertiesCount; Assert.Equal(42, settings.ReadWriteTimeout); ++checkedPropertiesCount; Assert.Equal("ClickHouse.NetCore Tests", settings.ClientName); ++checkedPropertiesCount; Assert.Equal(new ClickHouseVersion(3, 2, 1), settings.ClientVersion); ++checkedPropertiesCount; Assert.Equal(!ClickHouseConnectionStringBuilder.DefaultCompress, settings.Compress); ++checkedPropertiesCount; Assert.Equal(123, settings.CommandTimeout); ++checkedPropertiesCount; Assert.Equal(ClickHouseTlsMode.Require, settings.TlsMode); ++checkedPropertiesCount; Assert.Equal("/usr/local/share/ca-certificates/Yandex/YandexInternalRootCA.pem", settings.RootCertificate); ++checkedPropertiesCount; Assert.Equal(new byte[] { 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0 }, settings.ServerCertificateHash.ToArray()); ++checkedPropertiesCount; Assert.Equal(ClickHouseParameterMode.Interpolate, settings.ParametersMode); ++checkedPropertiesCount; Assert.Equal("unlimited", settings.QuotaKey); ++checkedPropertiesCount; Assert.Equal(checkedPropertiesCount, settings.GetType().GetProperties().Length); } [Fact] public void InvalidConnectionString() { Assert.Throws(() => new ClickHouseConnectionStringBuilder("Host=127.0.0.1;User=anonymous;Timeout:1337")); } [Fact] public void ValidConnectionStringWithInvalidProperty() { var ex = Assert.Throws(() => new ClickHouseConnectionStringBuilder("Host=127.0.0.1;User=anonymous;Timeout=1337;Server=clickhouse.example.com")); Assert.Equal("keyword", ex.ParamName); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseConnectionTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2021, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Data; using System.Linq; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Protocol; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseConnectionTests : ClickHouseTestsBase { [Fact] public async Task Probe() { // This test checks the version of the ClickHouse server specified in the connection string. // Its only purpose is to check if the CI pipeline has passed suitable connection settings. await using var cn = await OpenConnectionAsync(); Assert.NotNull(cn.ServerInfo); Assert.InRange(cn.ServerInfo.ServerRevision, ClickHouseProtocolRevisions.CurrentRevision, int.MaxValue); } [Fact] public void ShouldUnwrapConnectionOpenExceptions() { var sb = new ClickHouseConnectionStringBuilder { Host = "none.example.com" }; using var conn = new ClickHouseConnection(sb); var exception = Assert.ThrowsAny(() => conn.Open()); Assert.Equal(SocketError.HostNotFound, exception.SocketErrorCode); } [Fact] public async Task CanConnectWithUserAndPassword() { var settings = GetDefaultConnectionSettings(); settings = new ClickHouseConnectionStringBuilder(settings) { Database = null }.BuildSettings(); Assert.False(string.IsNullOrEmpty(settings.User)); Assert.Null(settings.Database); await using var conn = new ClickHouseConnection(settings); await conn.OpenAsync(); Assert.Equal(string.Empty, conn.Database); var currentUser = await conn.CreateCommand("select currentUser()").ExecuteScalarAsync(); var currentDb = await conn.CreateCommand("select currentDatabase()").ExecuteScalarAsync(); Assert.Equal(settings.User, currentUser); Assert.False(string.IsNullOrEmpty(currentDb)); } [Fact] public async Task TryPing() { await using var cn = new ClickHouseConnection(GetDefaultConnectionSettings()); var ex = await Assert.ThrowsAnyAsync(() => cn.TryPingAsync()); Assert.Equal(ClickHouseErrorCodes.ConnectionClosed, ex.ErrorCode); await cn.OpenAsync(); Assert.True(await cn.TryPingAsync()); var cmd = cn.CreateCommand("SELECT * FROM system.one"); Assert.True(await cn.TryPingAsync()); await using (var reader = await cmd.ExecuteReaderAsync()) { Assert.False(await cn.TryPingAsync()); Assert.True(await reader.ReadAsync()); Assert.False(await cn.TryPingAsync()); Assert.False(await reader.ReadAsync()); Assert.True(await cn.TryPingAsync()); } await using (await cmd.ExecuteReaderAsync()) { Assert.False(await cn.TryPingAsync()); } Assert.True(await cn.TryPingAsync()); await WithTemporaryTable( "ping", "id Int32", async (_, tableName) => { await using (var writer = await cn.CreateColumnWriterAsync($"INSERT INTO {tableName} VALUES", CancellationToken.None)) { Assert.False(await cn.TryPingAsync()); await writer.EndWriteAsync(CancellationToken.None); Assert.True(await cn.TryPingAsync()); } Assert.True(await cn.TryPingAsync()); await using (await cn.CreateColumnWriterAsync($"INSERT INTO {tableName} VALUES", CancellationToken.None)) { Assert.False(await cn.TryPingAsync()); } Assert.True(await cn.TryPingAsync()); }); } [Fact] public async Task OpenConnectionInParallel() { // Only one thread should open the connection and other threads should fail int counter = 0; var settings = GetDefaultConnectionSettings(); await using var connection = new ClickHouseConnection(settings); connection.StateChange += OnStateChanged; var tcs = new TaskCompletionSource(); var tasks = Enumerable.Range(0, 32).Select(_ => Task.Run(Open)).ToList(); tcs.SetResult(true); await Assert.ThrowsAnyAsync(() => Task.WhenAll(tasks)); int notFailedTaskCount = 0; foreach (var task in tasks) { if (task.Exception == null) { ++notFailedTaskCount; continue; } var aggrEx = Assert.IsAssignableFrom(task.Exception); Assert.Single(aggrEx.InnerExceptions); var ex = Assert.IsAssignableFrom(aggrEx.InnerExceptions[0]); Assert.Equal(ClickHouseErrorCodes.InvalidConnectionState, ex.ErrorCode); } // There should be at least one completed task and several failed tasks Assert.True(notFailedTaskCount > 0 && notFailedTaskCount < tasks.Count); Assert.Equal(ConnectionState.Open, connection.State); Assert.Equal(0x10001, counter); void OnStateChanged(object sender, StateChangeEventArgs e) { Assert.Same(sender, connection); if (e.OriginalState == ConnectionState.Closed && e.CurrentState == ConnectionState.Connecting) Interlocked.Increment(ref counter); if (e.OriginalState == ConnectionState.Connecting && e.CurrentState == ConnectionState.Open) Interlocked.Add(ref counter, 0x10000); } async Task Open() { await tcs.Task; await connection.OpenAsync(); } } [Theory] [InlineData(ConnectionState.Connecting, ConnectionState.Closed)] [InlineData(ConnectionState.Open, ConnectionState.Open)] [InlineData(ConnectionState.Closed, ConnectionState.Closed)] public async Task HandleCallbackException(ConnectionState throwOnState, ConnectionState expectedState) { var settings = GetDefaultConnectionSettings(); await using var connection = new ClickHouseConnection(settings); connection.StateChange += OnStateChanged; var ex = await Assert.ThrowsAnyAsync(async () => { await connection.OpenAsync(); await connection.CloseAsync(); }); Assert.Equal(expectedState, connection.State); Assert.Equal(ClickHouseErrorCodes.CallbackError, ex.ErrorCode); Assert.NotNull(ex.InnerException); Assert.Equal("You shall not pass!", ex.InnerException!.Message); void OnStateChanged(object sender, StateChangeEventArgs e) { Assert.Same(sender, connection); if (e.CurrentState == throwOnState) throw new Exception("You shall not pass!"); } } [Fact] public async Task HandleDoubleCallbackException() { var settings = GetDefaultConnectionSettings(); await using var connection = new ClickHouseConnection(settings); connection.StateChange += OnStateChanged; var ex = await Assert.ThrowsAnyAsync(() => connection.OpenAsync()); Assert.Equal(ConnectionState.Closed, connection.State); Assert.Equal(2, ex.InnerExceptions.Count); Assert.Equal("You shall not pass!", ex.InnerExceptions[0].Message); Assert.Equal("How dare you!", ex.InnerExceptions[1].Message); void OnStateChanged(object sender, StateChangeEventArgs e) { Assert.Same(sender, connection); if (e.CurrentState == ConnectionState.Closed) throw new Exception("How dare you!"); if (e.CurrentState == ConnectionState.Connecting) throw new Exception("You shall not pass!"); } } [Fact] public async Task DisposeCallback() { bool disposed = false; await using(var connection=await OpenConnectionAsync()) { connection.Disposed += OnDisposed; } Assert.True(disposed); void OnDisposed(object? sender, EventArgs eventArgs) { disposed = true; } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseDataReaderTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2024, 2026 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Types; using System; using System.Data; using System.Globalization; using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseDataReaderTests : ClickHouseTestsBase { [Fact] public async Task SimpleReader() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select 42 as answer"; await using var reader = await cmd.ExecuteReaderAsync(CancellationToken.None); var columnIndex = reader.GetOrdinal("answer"); Assert.True(await reader.ReadAsync(CancellationToken.None)); var value = reader.GetByte(columnIndex); Assert.False(await reader.ReadAsync()); Assert.Equal(42, value); } [Fact] public async Task CloseReaderWithoutReading() { await using var cn = await OpenConnectionAsync(); await using (var cmd = cn.CreateCommand("SELECT * FROM system.numbers")) { await using (var reader = await cmd.ExecuteReaderAsync()) { ulong value = 0; while (value < 100 && await reader.ReadAsync()) value = reader.GetFieldValue(0); Assert.Equal(100, value); } await using var reader2 = await cmd.ExecuteReaderAsync(CancellationToken.None); Assert.True(await reader2.ReadAsync()); var value2 = reader2.GetFieldValue(0); Assert.Equal(0, value2); } await using var cmd2 = cn.CreateCommand("SELECT * FROM system.one"); await (await cmd2.ExecuteReaderAsync()).DisposeAsync(); cmd2.CommandText = "SELECT * FROM system.numbers LIMIT 10000 OFFSET 42"; await using var reader3 = await cmd2.ExecuteReaderAsync(); Assert.True(await reader3.ReadAsync()); var value3 = reader3.GetFieldValue(0); Assert.Equal(42, value3); } [Fact] public async Task TotalsWithNextResult() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select x, sum(y) as v from (SELECT number%2 + 1 as x, number as y FROM numbers(10)) group by x with totals;"; using var reader = await cmd.ExecuteReaderAsync(); Assert.Equal(ClickHouseDataReaderState.Data, reader.State); ulong rowsTotal = 0; while (reader.Read()) { Assert.Equal(ClickHouseDataReaderState.Data, reader.State); rowsTotal += reader.GetFieldValue(1); } Assert.Equal(ClickHouseDataReaderState.NextResultPending, reader.State); var hasTotals = reader.NextResult(); Assert.True(hasTotals); Assert.Equal(ClickHouseDataReaderState.Totals, reader.State); Assert.True(reader.Read()); Assert.Equal(ClickHouseDataReaderState.Totals, reader.State); var total = reader.GetFieldValue(1); Assert.Equal(rowsTotal, total); Assert.False(reader.Read()); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); } [Fact] public async Task ExtremesWithNextResult() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.Extremes = true; cmd.CommandText = "select x, sum(y) as v from (SELECT number%2 + 1 as x, number as y FROM numbers(10)) group by x;"; using var reader = await cmd.ExecuteReaderAsync(); Assert.Equal(ClickHouseDataReaderState.Data, reader.State); ulong minX = ulong.MaxValue, maxX = ulong.MinValue, minSum = ulong.MaxValue, maxSum = ulong.MinValue; while (reader.Read()) { Assert.Equal(ClickHouseDataReaderState.Data, reader.State); var x = reader.GetFieldValue(0); var sum = reader.GetFieldValue(1); minX = Math.Min(minX, x); maxX = Math.Max(maxX, x); minSum = Math.Min(minSum, sum); maxSum = Math.Max(maxSum, sum); } Assert.Equal(ClickHouseDataReaderState.NextResultPending, reader.State); Assert.True(reader.NextResult()); Assert.True(reader.Read()); var extremeX = reader.GetFieldValue(0); var extremeSum = reader.GetFieldValue(1); Assert.Equal(minX, extremeX); Assert.Equal(minSum, extremeSum); Assert.True(reader.Read()); extremeX = reader.GetFieldValue(0); extremeSum = reader.GetFieldValue(1); Assert.Equal(maxX, extremeX); Assert.Equal(maxSum, extremeSum); Assert.False(reader.Read()); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); } [Fact] public async Task TotalsAndExtremes() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.Extremes = true; cmd.CommandText = "select x, sum(y) as v from (SELECT number%2 + 1 as x, number as y FROM numbers(10)) group by x with totals;"; using var reader = await cmd.ExecuteReaderAsync(); Assert.Equal(ClickHouseDataReaderState.Data, reader.State); ulong rowsTotal = 0; ulong minX = ulong.MaxValue, maxX = ulong.MinValue, minSum = ulong.MaxValue, maxSum = ulong.MinValue; while (reader.Read()) { Assert.Equal(ClickHouseDataReaderState.Data, reader.State); var x = reader.GetFieldValue(0); var sum = reader.GetFieldValue(1); rowsTotal += sum; minX = Math.Min(minX, x); maxX = Math.Max(maxX, x); minSum = Math.Min(minSum, sum); maxSum = Math.Max(maxSum, sum); } Assert.Equal(ClickHouseDataReaderState.NextResultPending, reader.State); var hasTotals = reader.NextResult(); Assert.True(hasTotals); Assert.Equal(ClickHouseDataReaderState.Totals, reader.State); Assert.True(reader.Read()); Assert.Equal(ClickHouseDataReaderState.Totals, reader.State); var total = reader.GetFieldValue(1); Assert.Equal(rowsTotal, total); Assert.False(reader.Read()); Assert.Equal(ClickHouseDataReaderState.NextResultPending, reader.State); Assert.True(reader.NextResult()); Assert.True(reader.Read()); var extremeX = reader.GetFieldValue(0); var extremeSum = reader.GetFieldValue(1); Assert.Equal(minX, extremeX); Assert.Equal(minSum, extremeSum); Assert.True(reader.Read()); extremeX = reader.GetFieldValue(0); extremeSum = reader.GetFieldValue(1); Assert.Equal(maxX, extremeX); Assert.Equal(maxSum, extremeSum); Assert.False(reader.Read()); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); } [Theory] [InlineData(ClickHouseDataReaderState.Data, 3)] [InlineData(ClickHouseDataReaderState.Extremes, 2)] [InlineData(ClickHouseDataReaderState.Totals, 1)] public async Task SkipNextResult(ClickHouseDataReaderState readBlock, int expectedCount) { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.Extremes = true; cmd.CommandText = "select x, sum(y) as v from (SELECT number%3 + 1 as x, number as y FROM numbers(10)) group by x with totals;"; await using var reader = await cmd.ExecuteReaderAsync(); Assert.Equal(ClickHouseDataReaderState.Data, reader.State); do { switch (reader.State) { case ClickHouseDataReaderState.Data: case ClickHouseDataReaderState.Totals: case ClickHouseDataReaderState.Extremes: if (reader.State == readBlock) break; continue; default: Assert.True(false, $"Unexpected state: {reader.State}."); break; } int count = 0; while (await reader.ReadAsync()) ++count; Assert.False(await reader.ReadAsync()); Assert.Equal(expectedCount, count); Assert.True(reader.State == ClickHouseDataReaderState.NextResultPending || reader.State == ClickHouseDataReaderState.Closed); } while (await reader.NextResultAsync()); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); } [Fact] public async Task CommandBehaviorSchemaOnly() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT * FROM system.numbers"); await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly); Assert.False(await reader.ReadAsync()); Assert.False(await reader.ReadAsync()); Assert.False(await reader.NextResultAsync()); Assert.False(await reader.ReadAsync()); Assert.Equal(1, reader.FieldCount); } [Fact] public async Task CommandBehaviorSingleRow() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT * FROM system.numbers"); await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SingleRow); Assert.Equal(1, reader.FieldCount); Assert.True(await reader.ReadAsync()); Assert.False(await reader.ReadAsync()); Assert.False(await reader.ReadAsync()); Assert.False(await reader.NextResultAsync()); Assert.False(await reader.ReadAsync()); } [Fact] public async Task CommandBehaviorSingleResult() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("select x, sum(y) as v from (SELECT number%7 + 1 as x, number as y FROM numbers(100)) group by x with totals;"); cmd.Extremes = true; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SingleResult); int count = 0; while (await reader.ReadAsync()) ++count; Assert.Equal(7, count); Assert.False(reader.Read()); Assert.NotEqual(ClickHouseDataReaderState.NextResultPending, reader.State); Assert.False(reader.NextResult()); Assert.False(reader.Read()); } [Fact] public async Task CommandBehaviorSequentialAccess() { // SequentialAccess is ignored await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT 41,42,43"); await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess); Assert.True(await reader.ReadAsync()); Assert.Equal(41, reader.GetInt32(0)); Assert.Equal(42, reader.GetInt32(1)); Assert.Equal(43, reader.GetInt32(2)); Assert.False(await reader.ReadAsync()); } [Fact] public async Task CommandBehaviorKeyInfo() { // KeyInfo is not supported await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand("SELECT 42"); var ex = await Assert.ThrowsAsync(() => cmd.ExecuteReaderAsync(CommandBehavior.KeyInfo)); Assert.Equal("behavior", ex.ParamName); } [Fact (Skip = "This test is flaky. The server doesn't always respond with profile events.")] public async Task ProfileEvents() { await using var cn = await OpenConnectionAsync(); const int expectedCount = 200_000; await using var cmd = cn.CreateCommand(string.Format(CultureInfo.InvariantCulture, "SELECT number as n, addSeconds('2000-01-01 00:00:00'::DateTime, n) as d FROM numbers({0})", expectedCount)); cmd.IgnoreProfileEvents = false; int count = 0, eventTableCount = 0; long selectedRows = 0; await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn("d", new ClickHouseColumnSettings(columnType: typeof(DateTime))); var startDate = new DateTimeOffset(2000, 1, 1, 0, 0, 0, -cn.GetServerTimeZone().GetUtcOffset(new DateTime(2000, 1, 1))); while(true) { Assert.Equal(ClickHouseDataReaderState.Data, reader.State); while (await reader.ReadAsync()) { var number = reader.GetUInt64(0); var dateObj = reader.GetValue(1); // Check that column settings applied var date = Assert.IsType(dateObj); Assert.Equal(startDate.AddSeconds(number).DateTime, date); ++count; } if (!await reader.NextResultAsync()) break; Assert.Equal(ClickHouseDataReaderState.ProfileEvents, reader.State); var typeColumnIdx = reader.GetOrdinal("type"); var nameColumnIdx = reader.GetOrdinal("name"); var valueColumnIdx = reader.GetOrdinal("value"); ++eventTableCount; while(await reader.ReadAsync()) { var name = reader.GetString(nameColumnIdx); if (name != "SelectedRows") continue; var type = reader.GetString(typeColumnIdx); var value = reader.GetInt64(valueColumnIdx); switch (type) { case "increment": selectedRows = value; break; default: Assert.True(false, $"Unexpected event type: {type}"); continue; } } if (!await reader.NextResultAsync()) break; }; Assert.True(eventTableCount > 1); Assert.Equal(expectedCount, count); Assert.True(selectedRows >= expectedCount); } [Fact] public async Task DateTimeKindForDate() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select cast('2022-01-01' as Date)"; await using var reader = await cmd.ExecuteReaderAsync(CancellationToken.None); Assert.True(await reader.ReadAsync(CancellationToken.None)); var value = reader.GetDateTime(0); Assert.False(await reader.ReadAsync()); Assert.Equal(2022, value.Year); Assert.Equal(1, value.Month); Assert.Equal(1, value.Day); Assert.Equal(0, value.Hour); Assert.Equal(0, value.Minute); Assert.Equal(0, value.Second); Assert.Equal(DateTimeKind.Unspecified, value.Kind); } [Fact] public async Task DateTimeKindForDate32() { await using var cn = await OpenConnectionAsync(); await using var cmd = cn.CreateCommand(); cmd.CommandText = "select cast('2022-01-01' as Date32)"; await using var reader = await cmd.ExecuteReaderAsync(CancellationToken.None); Assert.True(await reader.ReadAsync(CancellationToken.None)); var value = reader.GetDateTime(0); Assert.False(await reader.ReadAsync()); Assert.Equal(2022, value.Year); Assert.Equal(1, value.Month); Assert.Equal(1, value.Day); Assert.Equal(0, value.Hour); Assert.Equal(0, value.Minute); Assert.Equal(0, value.Second); Assert.Equal(DateTimeKind.Unspecified, value.Kind); } [Fact] public Task SparseSerializedColumns() { return WithTemporaryTable("sparse", Query, Test); static string Query(string tableName) => $@"CREATE TABLE {tableName} ( id UInt64, s Nullable(String), t Tuple(bool, String) ) ENGINE = MergeTree ORDER BY id SETTINGS ratio_of_defaults_for_sparse_serialization = 0.95"; static async Task Test(ClickHouseConnection cn, string tableName, CancellationToken ct) { var cmd = cn.CreateCommand(); const int expectedCount = 5_000_000; cmd.CommandText = $@"INSERT INTO {tableName} SELECT number, number % 25 = 0 ? toString(number) : '', tuple(number % 33 = 0, number % 99 = 0 ? toString(number) : '') FROM numbers({expectedCount})"; await cmd.ExecuteNonQueryAsync(ct); cmd.CommandText = $"SELECT id, s, t FROM {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(ct)) { int counter = 0; while (await reader.ReadAsync(ct)) { var id = reader.GetUInt64(0); var strVal = reader.GetString(1); var tuple = reader.GetFieldValue<(bool boolVal, string strVal)>(2); Assert.Equal(id % 25 == 0 ? id.ToString() : string.Empty, strVal); Assert.Equal(id % 33 == 0, tuple.boolVal); Assert.Equal(id % 99 == 0 ? id.ToString() : string.Empty, tuple.strVal); ++counter; } Assert.Equal(expectedCount, counter); } // Test skipping sparse values await cmd.ExecuteNonQueryAsync(ct); // Checking if the channel is in a valid state after skipping cmd.CommandText = "SELECT 21*2"; var answer = await cmd.ExecuteScalarAsync(ct); Assert.Equal(42, answer); } } [Fact] public async Task CustomColumnCast() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT 42::Float32"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumnReader(0, (double v) => new TestBox((decimal)v)); Assert.True(await reader.ReadAsync()); Assert.Equal(typeof(TestBox), reader.GetFieldType(0)); var res = reader.GetValue(0); Assert.IsType>(res); Assert.Equal(42m, ((TestBox)res).Unbox()); var box = reader.GetFieldValue>(0); Assert.Equal(42m, box.Unbox()); Assert.False(await reader.ReadAsync()); } [Fact] public async Task CustomNullableColumnCast() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT if(number=1, NULL, (number + 40)::Nullable(Float32)) AS c, c AS c_copy FROM numbers(1, 2)"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumnReader(0, (double v) => (decimal)v); reader.ConfigureColumnReader("c_copy", (float? v) => v == null ? (TestBox?)null : new TestBox((int)v)); Assert.Equal(typeof(decimal), reader.GetFieldType(0)); Assert.Equal(typeof(TestBox), reader.GetFieldType(1)); Assert.True(await reader.ReadAsync()); Assert.True(reader.IsDBNull(0)); Assert.True(reader.IsDBNull(1)); var res = reader.GetValue(0); Assert.IsType(res); var resCopy = reader.GetValue(1); Assert.IsType(resCopy); var box = reader.GetFieldValue>(1, null); Assert.Null(box); Assert.True(await reader.ReadAsync()); Assert.False(reader.IsDBNull(0)); Assert.False(reader.IsDBNull(1)); res = reader.GetValue(0); Assert.IsType(res); Assert.Equal(42m, (decimal)res); var altReinterpreted = reader.GetDouble(1); Assert.Equal(42, altReinterpreted); resCopy = reader.GetValue(1); Assert.IsType>(resCopy); Assert.Equal(42, ((TestBox)resCopy).Unbox()); box = reader.GetFieldValue?>(1); Assert.NotNull(box); Assert.Equal(42, box.Value.Unbox()); box = reader.GetFieldValue>(1); Assert.NotNull(box); Assert.Equal(42, box.Value.Unbox()); Assert.False(await reader.ReadAsync()); } [Fact] public async Task CustomObjecctColumnCast() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT 42::Float32"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumnReader(0, (object _) => 24); Assert.Equal(typeof(int), reader.GetFieldType(0)); Assert.True(await reader.ReadAsync()); var res = reader.GetValue(0); Assert.IsType(res); Assert.Equal(24, (int)res); Assert.False(await reader.ReadAsync()); } [Fact] public async Task ValidColumnReconfiguration() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT 42::Float32 AS col0, 42::Int32 AS col1, 42 AS col2"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn(0, new ClickHouseColumnSettings(Encoding.UTF8)); reader.ConfigureColumnReader("col0", (float f) => (double)f); reader.ConfigureColumnReader(2, (byte b) => (double)b); reader.ConfigureColumn("col2", new ClickHouseColumnSettings(enumConverter: new ClickHouseEnumConverter())); Assert.True(await reader.ReadAsync()); var result = new object[3]; Assert.Equal(3, reader.GetValues(result)); Assert.Equal(42d, result[0]); Assert.Equal(42, result[1]); Assert.Equal(42d, result[2]); Assert.False(await reader.ReadAsync()); } [Fact] public async Task NullableObjectColumnReconfiguration() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT number AS n, multiIf(number%15==0, 'fizzbuzz', number%5==0, 'buzz', number%3==0, 'fizz', NULL) AS s FROM numbers(1, 15)"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumnReader("s", (string s) => s.ToUpperInvariant()); Assert.Equal(typeof(ulong), reader.GetFieldType(0)); Assert.Equal(typeof(string), reader.GetFieldType(1)); int count = 0; var result = new object[2]; while(await reader.ReadAsync()) { Assert.Equal(2, reader.GetValues(result)); var n = Assert.IsType(result[0]); string? expected = null; if (n % 3 == 0) expected = "FIZZ"; if (n % 5 ==0) expected = expected == null ? "BUZZ" : "FIZZBUZZ"; if (expected == null) { Assert.IsType(result[1]); Assert.True(reader.IsDBNull(1)); var value = reader.GetValue(1); Assert.IsType(value); var strNullable = reader.GetFieldValue(1, "NONE"); Assert.Equal("NONE", strNullable); } else { var str = Assert.IsType(result[1]); Assert.Equal(expected, str); Assert.False(reader.IsDBNull(1)); str = reader.GetString(1); Assert.Equal(expected, str); var strNullable = reader.GetFieldValue(1, "NONE"); Assert.Equal(expected, strNullable); } count++; } Assert.Equal(15, count); } [Fact] public async Task InavlidColumnReconfiguration() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT 42::Float32 AS col0, 42::Float64 AS col1, 42 AS col2"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureDataReader(new ClickHouseColumnSettings(typeof(double))); var err = Assert.Throws(() => reader.ConfigureColumnReader(0, (double v) => v + 1)); Assert.Equal(ClickHouseErrorCodes.InvalidColumnSettings, err.ErrorCode); reader.ConfigureColumn("col2", new ClickHouseColumnSettings()); err = Assert.Throws(() => reader.ConfigureColumnReader("col2", (byte v) => (string?)null)); Assert.Equal(ClickHouseErrorCodes.CallbackError, err.ErrorCode); reader.ConfigureColumnReader("col2", (byte? v) => v / 2); err = Assert.Throws(() => reader.ConfigureColumn(2, new ClickHouseColumnSettings(typeof(int)))); Assert.Equal(ClickHouseErrorCodes.InvalidColumnSettings, err.ErrorCode); err = Assert.Throws(() => reader.ConfigureDataReader(new ClickHouseColumnSettings(typeof(int)))); Assert.Equal(ClickHouseErrorCodes.InvalidColumnSettings, err.ErrorCode); Assert.True(await reader.ReadAsync()); err = Assert.Throws(() => reader.ConfigureColumnReader("col0", (float v) => (int)v)); Assert.Equal(ClickHouseErrorCodes.DataReaderError, err.ErrorCode); var result = new object[3]; Assert.Equal(3, reader.GetValues(result)); Assert.Equal(42d, result[0]); Assert.Equal(42d, result[1]); Assert.Equal(21, result[2]); Assert.False(await reader.ReadAsync()); } [Fact] public async Task ExecuteDdlQuery() { await WithTemporaryTable("reader_drop", "dummy Int32", Test); static async Task Test(ClickHouseConnection cn, string tableName) { var cmd = cn.CreateCommand($"DROP TABLE {tableName}"); await using var reader = await cmd.ExecuteReaderAsync(CancellationToken.None); Assert.True(reader.IsClosed); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); Assert.Equal(0, reader.FieldCount); Assert.False(await reader.ReadAsync()); } } [Fact] public async Task ExecuteCreateInsertFromSelect() { await WithTemporaryTable("reader_from_select", "dummy Int32", Test); static async Task Test(ClickHouseConnection cn, string tableName) { var cmd = cn.CreateCommand($"INSERT INTO {tableName} SELECT number FROM system.numbers LIMIT 42"); await using (var reader = await cmd.ExecuteReaderAsync(CancellationToken.None)) { Assert.True(reader.IsClosed); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); Assert.Equal(0, reader.FieldCount); Assert.False(await reader.ReadAsync()); } cmd.CommandText = $"DROP TABLE {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(CancellationToken.None)) { Assert.True(reader.IsClosed); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); Assert.Equal(0, reader.FieldCount); Assert.False(await reader.ReadAsync()); } cmd.CommandText = $"CREATE TABLE {tableName} ENGINE = Memory AS SELECT number FROM system.numbers LIMIT 100"; await using (var reader = await cmd.ExecuteReaderAsync(CancellationToken.None)) { Assert.True(reader.IsClosed); Assert.Equal(ClickHouseDataReaderState.Closed, reader.State); Assert.Equal(0, reader.FieldCount); Assert.False(await reader.ReadAsync()); } } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseDbProviderFactoryTests.cs ================================================ #region License Apache 2.0 /* Copyright 2020, 2023 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Data; using System.Data.Common; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseDbProviderFactoryTests { private static readonly DbProviderFactory Factory = new ClickHouseDbProviderFactory(); [Fact] public void SupportedFeatures() { Assert.False(Factory.CanCreateCommandBuilder); Assert.False(Factory.CanCreateDataAdapter); Assert.False(Factory.CanCreateDataSourceEnumerator); var p = Factory.CreateParameter(); Assert.IsAssignableFrom(p); var cmd = Factory.CreateCommand(); Assert.NotNull(cmd); Assert.IsAssignableFrom(cmd); cmd.Dispose(); var cn = Factory.CreateConnection(); Assert.NotNull(cn); Assert.IsAssignableFrom(cn); cn.Dispose(); var sb = Factory.CreateConnectionStringBuilder(); Assert.IsAssignableFrom(sb); } [Theory] [MemberData(nameof(ClickHouseTestsBase.ParameterModes), MemberType = typeof(ClickHouseTestsBase))] public void CreateCommand(ClickHouseParameterMode parameterMode) { var connectionString = ConnectionSettingsHelper.GetConnectionString(csb => csb.ParametersMode = parameterMode); var sb = Factory.CreateConnectionStringBuilder(); Assert.NotNull(sb); sb.ConnectionString = connectionString; using var connection = Factory.CreateConnection(); Assert.NotNull(connection); using var cmd = Factory.CreateCommand(); Assert.NotNull(cmd); var parameterName = cmd.CreateParameter().ParameterName; cmd.Parameters.RemoveAt(parameterName); Assert.Empty(cmd.Parameters); var parameter = Factory.CreateParameter(); Assert.NotNull(parameter); cmd.CommandText = $"SELECT * FROM system.one WHERE dummy < {parameterName}"; cmd.Parameters.Add(parameter); parameter.ParameterName = parameterName; parameter.DbType = DbType.Int32; parameter.Value = int.MaxValue; connection.ConnectionString = sb.ToString(); connection.Open(); cmd.Connection = connection; var result = cmd.ExecuteScalar(); Assert.Equal((byte) 0, result); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseParameterCollectionTests.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Data; using System.Linq; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseParameterCollectionTests { [Fact] public void AddParameter() { var parameterNames = new[] {"abc", "{abc}", "@abc"}; foreach (var name in parameterNames) { var collection = new ClickHouseParameterCollection(); var parameter = new ClickHouseParameter(name); collection.Add(parameter); Assert.Single(collection); Assert.Same(parameter, collection[0]); foreach (var altName in parameterNames) { Assert.True(collection.Contains(altName)); Assert.True(collection.TryGetValue(altName, out var collectionParameter)); Assert.Same(parameter, collectionParameter); Assert.Same(collection, parameter.Collection); } } } [Fact] public void ChangeParameterName() { var collection = new ClickHouseParameterCollection(); var parameter = new ClickHouseParameter("p1"); Assert.Null(parameter.Collection); collection.Add(parameter); Assert.Same(collection, parameter.Collection); parameter.ParameterName = "{p1}"; Assert.Same(collection, parameter.Collection); Assert.True(collection.TryGetValue(parameter.ParameterName, out var collectionParameter)); Assert.Same(parameter, collectionParameter); parameter.ParameterName = "param123"; Assert.Same(collection, parameter.Collection); Assert.True(collection.TryGetValue(parameter.ParameterName, out collectionParameter)); Assert.Same(parameter, collectionParameter); parameter.ParameterName = "p1"; Assert.Same(collection, parameter.Collection); Assert.True(collection.TryGetValue(parameter.ParameterName, out collectionParameter)); Assert.Same(parameter, collectionParameter); parameter.ParameterName = "P1"; Assert.Same(collection, parameter.Collection); Assert.True(collection.TryGetValue(parameter.ParameterName, out collectionParameter)); Assert.Same(parameter, collectionParameter); } [Fact] public void RemoveParameter() { var collection = new ClickHouseParameterCollection(); var parameters = new[] {new ClickHouseParameter("p1"), new ClickHouseParameter("p2"), new ClickHouseParameter("p3")}; collection.AddRange(parameters); Assert.Equal(parameters.Length, collection.Count); Assert.False(collection.Remove("p4", out var removedParameter)); Assert.Null(removedParameter); Assert.Equal(parameters.Length, collection.Count); Assert.False(collection.Remove("p4")); Assert.Equal(parameters.Length, collection.Count); collection.RemoveAt("p4"); Assert.Equal(parameters.Length, collection.Count); Assert.True(collection.Remove("p1")); Assert.Equal(2, collection.Count); Assert.Null(parameters[0].Collection); Assert.Same(parameters[1], collection[0]); Assert.Same(parameters[2], collection[1]); Assert.True(collection.Remove("p3", out removedParameter)); Assert.Equal(1, collection.Count); Assert.Null(parameters[2].Collection); Assert.Same(parameters[2], removedParameter); Assert.Same(collection[0], parameters[1]); collection.RemoveAt("p2"); Assert.Equal(0, collection.Count); Assert.Null(parameters[1].Collection); collection.AddRange(parameters.Reverse().ToArray()); Assert.Equal(parameters.Length, collection.Count); collection.RemoveAt(0); Assert.Equal(2, collection.Count); Assert.Null(parameters[2].Collection); Assert.Same(parameters[1], collection[0]); Assert.Same(parameters[0], collection[1]); collection.Remove((object)parameters[2]); Assert.Equal(2, collection.Count); Assert.False(collection.Remove(parameters[2])); Assert.Equal(2, collection.Count); Assert.True(collection.Remove(parameters[1])); Assert.Equal(1, collection.Count); Assert.Null(parameters[1].Collection); Assert.Same(collection[0], parameters[0]); } [Fact] public void InsertParameter() { var collection = new ClickHouseParameterCollection(); collection.Insert(0, (object) new ClickHouseParameter("p1")); Assert.Equal(1, collection.Count); Assert.Equal("p1", collection[0].ParameterName); Assert.Same(collection, collection[0].Collection); collection.Insert(0, new ClickHouseParameter("{p2}")); Assert.Equal(2, collection.Count); Assert.Equal("{p2}", collection[0].ParameterName); Assert.Same(collection, collection[0].Collection); Assert.Equal("p1", collection[1].ParameterName); Assert.Same(collection, collection[1].Collection); collection.Insert(2, (object) new ClickHouseParameter("@p3")); Assert.Equal(3, collection.Count); Assert.Equal("{p2}", collection[0].ParameterName); Assert.Same(collection, collection[0].Collection); Assert.Equal("p1", collection[1].ParameterName); Assert.Same(collection, collection[1].Collection); Assert.Equal("@p3", collection[2].ParameterName); Assert.Same(collection, collection[2].Collection); collection.Insert(1, new ClickHouseParameter("p4")); Assert.Equal(4, collection.Count); Assert.Equal("{p2}", collection[0].ParameterName); Assert.Same(collection, collection[0].Collection); Assert.Equal("p4", collection[1].ParameterName); Assert.Same(collection, collection[1].Collection); Assert.Equal("p1", collection[2].ParameterName); Assert.Same(collection, collection[2].Collection); Assert.Equal("@p3", collection[3].ParameterName); Assert.Same(collection, collection[3].Collection); } [Fact] public void ReplaceParameter() { var collection = new ClickHouseParameterCollection(); var parameters = new[] {"p1", "p2", "p3", "p4"}.Select(name => collection.AddWithValue(name, null, DbType.String)).ToArray(); Assert.Equal(4, collection.Count); var newParam = new ClickHouseParameter("p5"); collection[2] = newParam; Assert.Equal(4, collection.Count); Assert.Equal("p1", collection[0].ParameterName); Assert.Same(collection, collection[0].Collection); Assert.Equal("p2", collection[1].ParameterName); Assert.Same(collection, collection[1].Collection); Assert.Same(newParam, collection[2]); Assert.Same(collection, collection[2].Collection); Assert.Equal("p4", collection[3].ParameterName); Assert.Same(collection, collection[3].Collection); Assert.Null(parameters[2].Collection); collection["{p5}"] = parameters[2]; Assert.Equal(4, collection.Count); for (int i = 0; i < parameters.Length; i++) { Assert.Same(parameters[i], collection[i]); Assert.Same(collection, parameters[i].Collection); } Assert.Null(newParam.Collection); collection["p1"] = parameters[0]; Assert.Equal(4, collection.Count); for (int i = 0; i < parameters.Length; i++) { Assert.Same(parameters[i], collection[i]); Assert.Same(collection, parameters[i].Collection); } collection["{p5}"] = newParam; Assert.Equal(5, collection.Count); for (int i = 0; i < parameters.Length; i++) { Assert.Same(parameters[i], collection[i]); Assert.Same(collection, parameters[i].Collection); } Assert.Same(newParam, collection[4]); Assert.Same(collection, newParam.Collection); var ex = Assert.Throws(() => collection["p6"] = new ClickHouseParameter("p7")); Assert.Equal(5, collection.Count); Assert.Equal("parameterName", ex.ParamName); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseParameterTests.cs ================================================ #region License Apache 2.0 /* Copyright 2021-2022 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Data; using System.Linq.Expressions; using System.Net; using System.Reflection; using System.Text; using Octonica.ClickHouseClient.Utils; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseParameterTests { private static readonly ReadOnlyDictionary> ParameterPublicProperties; static ClickHouseParameterTests() { var properties = new Dictionary>(); foreach (var property in typeof(ClickHouseParameter).GetProperties(BindingFlags.Public | BindingFlags.Instance)) { if (property.GetMethod == null || property.SetMethod == null || !property.GetMethod.IsPublic || !property.SetMethod.IsPublic) continue; var comparer = TypeDispatcher.Dispatch(property.PropertyType, new PropertyComparerDispatcher(property)); properties.Add(property.Name, comparer); } ParameterPublicProperties = new ReadOnlyDictionary>(properties); } [Fact] public void Clone() { var p1 = new ClickHouseParameter("p1") { Value = new[] { 42 }, ClickHouseDbType = ClickHouseDbType.VarNumeric, Precision = 19, Scale = 7, ParameterMode = ClickHouseParameterMode.Interpolate }; var p2 = p1.Clone(); Assert.NotSame(p1, p2); AssertParametersEqual(p1, p2); Assert.Equal("p1", p1.ParameterName); Assert.Equal(p1.Value, p2.Value); Assert.Equal(ClickHouseDbType.VarNumeric, p2.ClickHouseDbType); Assert.Equal(19, p2.Precision); Assert.Equal(7, p2.Scale); Assert.Equal(ClickHouseParameterMode.Interpolate, p2.ParameterMode); p2.TimeZone = TimeZoneInfo.Local; p2.Size = 35; p2.ArrayRank = 3; p2.StringEncoding = Encoding.ASCII; p2.IsNullable = true; var collection = new ClickHouseParameterCollection { p2 }; var p3 = p2.Clone(); Assert.NotSame(p2, p3); Assert.Null(p3.Collection); Assert.Same(collection, p2.Collection); AssertParametersEqual(p2, p3); Assert.Equal(TimeZoneInfo.Local, p3.TimeZone); Assert.Equal(35, p3.Size); Assert.Equal(3, p3.ArrayRank); Assert.Equal(Encoding.ASCII, p3.StringEncoding); Assert.True(p3.IsNullable); p2.SourceColumn = "some value"; p2.SourceColumnNullMapping = true; var p4 = p2.Clone(); Assert.NotSame(p2, p4); Assert.Null(p4.Collection); AssertParametersEqual(p2, p4); Assert.Equal("some value", p4.SourceColumn); Assert.True(p2.SourceColumnNullMapping); } [Fact] public void CopyTo() { var p1 = new ClickHouseParameter("p1") { ArrayRank = 150, ClickHouseDbType = ClickHouseDbType.IpV6, IsArray = true, IsNullable = true, Precision = 24, Scale = 255, Size = 123456, SourceColumn = "aaaaa", SourceColumnNullMapping = true, StringEncoding = Encoding.BigEndianUnicode, TimeZone = TimeZoneInfo.Utc, Value = 123.456m }; var collection = new ClickHouseParameterCollection { p1, new ClickHouseParameter("p2") }; var p2 = collection["p2"]; p1.CopyTo(p2); AssertParametersEqual(p1, p2, false); } [Fact] public void NotSupportedProperties() { var p = new ClickHouseParameter(); Assert.Throws(() => p.Direction = ParameterDirection.InputOutput); Assert.Throws(() => p.Direction = ParameterDirection.Output); Assert.Throws(() => p.Direction = ParameterDirection.ReturnValue); // Inherited behaviour p.SourceVersion = DataRowVersion.Current; Assert.Equal(DataRowVersion.Default, p.SourceVersion); p.SourceVersion = DataRowVersion.Original; Assert.Equal(DataRowVersion.Default, p.SourceVersion); p.SourceVersion = DataRowVersion.Proposed; Assert.Equal(DataRowVersion.Default, p.SourceVersion); } [Fact] public void TypeDetection() { var testData = new (object? value, ClickHouseDbType expectedType, bool expectedNullable, int expectedArrayRank)[] { (null, ClickHouseDbType.Nothing, true, 0), (DBNull.Value, ClickHouseDbType.Nothing, true, 0), (new[] {DBNull.Value}, ClickHouseDbType.Nothing, true, 1), (true, ClickHouseDbType.Boolean, false, 0), ((byte) 1, ClickHouseDbType.Byte, false, 0), ((sbyte) 1, ClickHouseDbType.SByte, false, 0), ((short) 1, ClickHouseDbType.Int16, false, 0), ((ushort) 1, ClickHouseDbType.UInt16, false, 0), ((int) 1, ClickHouseDbType.Int32, false, 0), ((uint) 1, ClickHouseDbType.UInt32, false, 0), ((long) 1, ClickHouseDbType.Int64, false, 0), ((ulong) 1, ClickHouseDbType.UInt64, false, 0), ((float) 1, ClickHouseDbType.Single, false, 0), ((double) 1, ClickHouseDbType.Double, false, 0), ((decimal) 1, ClickHouseDbType.Decimal, false, 0), (DateTime.Now, ClickHouseDbType.DateTime, false, 0), (DateTimeOffset.Now, ClickHouseDbType.DateTime, false, 0), (Guid.Empty, ClickHouseDbType.Guid, false, 0), (new IPAddress(new byte[] {127, 0, 0, 1}), ClickHouseDbType.IpV4, false, 0), (IPAddress.Parse("2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d"), ClickHouseDbType.IpV6, false, 0), (string.Empty, ClickHouseDbType.String, false, 0), (new[] {'!'}, ClickHouseDbType.String, false, 0), (string.Empty.AsMemory(), ClickHouseDbType.String, false, 0), (new[] {'!'}.AsMemory(), ClickHouseDbType.String, false, 0), (new byte[0], ClickHouseDbType.Byte, false, 1), (new byte[0].AsMemory(), ClickHouseDbType.Byte, false, 1), ((ReadOnlyMemory) new byte[0].AsMemory(), ClickHouseDbType.Byte, false, 1), (new Guid?[0, 0, 0], ClickHouseDbType.Guid, true, 3), }; var p = new ClickHouseParameter("p"); foreach(var testCase in testData) { p.Value = testCase.value; Assert.Equal(testCase.expectedType, p.ClickHouseDbType); Assert.Equal(testCase.expectedArrayRank, p.ArrayRank); Assert.Equal(testCase.expectedArrayRank > 0, p.IsArray); Assert.Equal(testCase.expectedNullable, p.IsNullable); } } private static void AssertParametersEqual(ClickHouseParameter expected, ClickHouseParameter actual, bool compareName = true) { foreach (var pair in ParameterPublicProperties) { if (pair.Key == nameof(ClickHouseParameter.ParameterName) && !compareName) continue; pair.Value(expected, actual); } } private class PropertyComparerDispatcher : ITypeDispatcher> { private readonly PropertyInfo _propertyInfo; public PropertyComparerDispatcher(PropertyInfo propertyInfo) { _propertyInfo = propertyInfo; } public Action Dispatch() { var expParam = Expression.Variable(typeof(ClickHouseParameter), "p"); var expGetValue = Expression.Property(expParam, _propertyInfo); var expLambda = Expression.Lambda>(expGetValue, expParam); var getValue = expLambda.Compile(); return (expected, actual) => { var expectedValue = getValue(expected); var actualValue = getValue(actual); Assert.Equal(expectedValue, actualValue); }; } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseTestsBase.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Tests { public abstract class ClickHouseTestsBase { private ClickHouseConnectionSettings? _settings; public static readonly IReadOnlyCollection ParameterModes = new[] { new object[] { ClickHouseParameterMode.Binary }, new object[] { ClickHouseParameterMode.Interpolate }, new object[] { ClickHouseParameterMode.Serialize } }; public ClickHouseConnectionSettings GetDefaultConnectionSettings(Action? updateSettings = null) { if (_settings != null) { return _settings; } _settings = ConnectionSettingsHelper.GetConnectionSettings(updateSettings); return _settings; } public async Task OpenConnectionAsync(ClickHouseConnectionSettings settings, CancellationToken cancellationToken) { ClickHouseConnection connection = new ClickHouseConnection(settings); await connection.OpenAsync(cancellationToken); return connection; } public Task OpenConnectionAsync(ClickHouseParameterMode parameterMode, CancellationToken cancellationToken = default) { return OpenConnectionAsync(builder => builder.ParametersMode = parameterMode, cancellationToken); } public async Task OpenConnectionAsync(Action? updateSettings = null, CancellationToken cancellationToken = default) { return await OpenConnectionAsync(GetDefaultConnectionSettings(updateSettings), cancellationToken); } public ClickHouseConnection OpenConnection(Action? updateSettings = null) { ClickHouseConnection connection = new ClickHouseConnection(GetDefaultConnectionSettings(updateSettings)); connection.Open(); return connection; } protected Task WithTemporaryTable(string tableNameSuffix, string columns, Func runTest, Action? updateSettings = null) { return WithTemporaryTable(tableNameSuffix, columns, (cn, tableName, _) => runTest(cn, tableName), updateSettings); } protected Task WithTemporaryTable( string tableNameSuffix, string columns, Func runTest, Action? updateSettings = null, Func? afterOpen = null, CancellationToken ct = default) { return WithTemporaryTable(tableNameSuffix, tableName => $"CREATE TABLE {tableName}({columns}) ENGINE=Memory", runTest, updateSettings, afterOpen, ct); } protected async Task WithTemporaryTable( string tableNameSuffix, Func makeCreateTableQuery, Func runTest, Action? updateSettings = null, Func? configure = null, CancellationToken ct = default) { var tableName = GetTempTableName(tableNameSuffix); try { await using var connection = await OpenConnectionAsync(updateSettings, ct); if (configure != null) await configure(connection, ct); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {tableName}"); await cmd.ExecuteNonQueryAsync(ct); var createTableQuery = makeCreateTableQuery(tableName); cmd = connection.CreateCommand(createTableQuery); await cmd.ExecuteNonQueryAsync(ct); await runTest(connection, tableName, ct); } finally { await using var connection = await OpenConnectionAsync(cancellationToken: CancellationToken.None); var cmd = connection.CreateCommand($"DROP TABLE IF EXISTS {tableName}"); await cmd.ExecuteNonQueryAsync(CancellationToken.None); } } protected virtual string GetTempTableName(string tableNameSuffix) { return $"clickhouse_client_test_{tableNameSuffix}"; } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ClickHouseTypeInfoTests.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Linq; using Octonica.ClickHouseClient.Types; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ClickHouseTypeInfoTests { [Theory] [InlineData("Nullable(Nothing)", "Nothing")] [InlineData("Nullable( String )", "String")] [InlineData("Nullable ( DateTime( 'Asia/Yekaterinburg' ) )", "DateTime('Asia/Yekaterinburg')")] public void NullableGenericArguments(string typeName, string baseTypeName) { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal(1, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); Assert.Equal(baseTypeName, typeInfo.GetGenericArgument(0).ComplexTypeName); Assert.Equal(baseTypeName, Assert.IsAssignableFrom(typeInfo.GetTypeArgument(0)).ComplexTypeName); } [Theory] [InlineData("LowCardinality(String)", "String")] [InlineData("LowCardinality( Decimal ( 28, 10 ))", "Decimal(28, 10)")] public void LowCardinalityGenericArguments(string typeName, string baseTypeName) { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal(1, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); Assert.Equal(baseTypeName, typeInfo.GetGenericArgument(0).ComplexTypeName); Assert.Equal(baseTypeName, Assert.IsAssignableFrom(typeInfo.GetTypeArgument(0)).ComplexTypeName); } [Fact] public void TupleGenericArguments() { var typeNames = new[] {"Decimal(19, 6)", "String", "Nullable(String)", "DateTime64(5, 'Europe/Prague')", "UInt8", "Int32", "Float32", "Enum8('a'=10, 'b'=20)", "UInt64"}; for (int i = 1; i <= typeNames.Length; i++) { var typeName = "Tuple(" + string.Join(',', typeNames.Take(i)) + ')'; var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal(i, typeInfo.GenericArgumentsCount); Assert.Equal(i, typeInfo.TypeArgumentsCount); for (int j = 0; j < i; j++) { IClickHouseTypeInfo baseType = typeInfo.GetGenericArgument(j); Assert.Equal(typeNames[j], baseType.ComplexTypeName); var typeArgument = typeInfo.GetTypeArgument(j); baseType = Assert.IsAssignableFrom(typeArgument); Assert.Equal(typeNames[j], baseType.ComplexTypeName); } } } [Theory] [InlineData("Array(Int32)", "Int32", null)] [InlineData("Array(Nullable(Int32))", "Nullable(Int32)", "Int32")] [InlineData("Array(Array(Nothing))", "Array(Nothing)", "Nothing")] public void ArrayGenericArguments(string typeName, string baseTypeName, string? baseBaseTypeName) { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal(1, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); var baseType = typeInfo.GetGenericArgument(0); Assert.Equal(baseTypeName, baseType.ComplexTypeName); Assert.Equal(baseTypeName, Assert.IsAssignableFrom(typeInfo.GetTypeArgument(0)).ComplexTypeName); if (baseBaseTypeName == null) { Assert.Equal(0, baseType.GenericArgumentsCount); return; } Assert.Equal(1, baseType.GenericArgumentsCount); Assert.Equal(1, baseType.TypeArgumentsCount); Assert.Equal(baseBaseTypeName, baseType.GetGenericArgument(0).ComplexTypeName); Assert.Equal(baseBaseTypeName, Assert.IsAssignableFrom(baseType.GetTypeArgument(0)).ComplexTypeName); } [Theory] [InlineData("Decimal32(5)", "Decimal32", 5, null)] [InlineData("Decimal64(1)", "Decimal64", 1, null)] [InlineData("Decimal128(9)", "Decimal128", 9, null)] [InlineData("Decimal(35, 10)", "Decimal", 35, 10)] public void DecimalTypeArguments(string typeName, string expectedTypeName, int firstArgument, int? secondArgument) { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); int expectedCount = 1 + (secondArgument == null ? 0 : 1); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(expectedTypeName, typeInfo.TypeName); Assert.Equal(expectedCount, typeInfo.TypeArgumentsCount); Assert.Equal(firstArgument, typeInfo.GetTypeArgument(0)); if (secondArgument != null) Assert.Equal(secondArgument, typeInfo.GetTypeArgument(1)); } [Fact] public void DateTimeTypeArguments() { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo("DateTime('Asia/Macau')"); Assert.Equal("DateTime", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); Assert.Equal("Asia/Macau", typeInfo.GetTypeArgument(0)); typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo("DateTime"); Assert.Equal("DateTime", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(0, typeInfo.TypeArgumentsCount); } [Fact] public void DateTime64TypeArguments() { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo("DateTime64(3, 'Africa/Addis_Ababa')"); Assert.Equal("DateTime64", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(2, typeInfo.TypeArgumentsCount); Assert.Equal(3, typeInfo.GetTypeArgument(0)); Assert.Equal("Africa/Addis_Ababa", typeInfo.GetTypeArgument(1)); typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo("DateTime64(5)"); Assert.Equal("DateTime64", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); Assert.Equal(5, typeInfo.GetTypeArgument(0)); } [Theory] [InlineData("Enum8('a' = 42)", new[] { "a" }, new sbyte[] { 42 })] [InlineData("Enum8('a' = 2, 'C' = -3,'b'=1)", new[] { "a", "C", "b" }, new sbyte[] { 2, -3, 1 })] [InlineData("Enum8('\\'a\\'' = -5, ' \\tescaped \\'value\\' ({[ ' = -9,'\\r\\n\\t\\d\\\\'= 18)", new[] { "'a'", " \tescaped 'value' ({[ ", "\r\n\t\\d\\" }, new sbyte[] { -5, -9, 18 })] public void Enum8TypeArguments(string typeName, string[] expectedKeys, sbyte[] expectedValues) { Assert.Equal(expectedKeys.Length, expectedValues.Length); var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal("Enum8", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(expectedKeys.Length, typeInfo.TypeArgumentsCount); for (int i = 0; i < expectedKeys.Length; i++) { var typeArgumentObj = typeInfo.GetTypeArgument(i); var typeArgument = Assert.IsType>(typeArgumentObj); Assert.Equal(typeArgument.Key, expectedKeys[i]); Assert.Equal(typeArgument.Value, expectedValues[i]); } } [Theory] [InlineData("Enum16('a' = 1024)", new[] { "a" }, new short[] { 1024 })] [InlineData("Enum16('a' = 8965, 'C' = 5,'b'=-3256)", new[] { "a", "C", "b" }, new short[] { 8965, 5, -3256 })] [InlineData("Enum16('\"a\"' = 31000 , '\\'\\\\e\\s\\c\\\\a\\p\\e\\d\\'' = -31000, '}])' = 42)", new[] { "\"a\"", @"'\e\s\c\a\p" + "\x1b" + @"\d'", "}])" }, new short[] { 31000, -31000, 42 })] public void Enum16TypeArguments(string typeName, string[] expectedKeys, short[] expectedValues) { Assert.Equal(expectedKeys.Length, expectedValues.Length); var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); Assert.Equal("Enum16", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(expectedKeys.Length, typeInfo.TypeArgumentsCount); for (int i = 0; i < expectedKeys.Length; i++) { var typeArgumentObj = typeInfo.GetTypeArgument(i); var typeArgument = Assert.IsType>(typeArgumentObj); Assert.Equal(typeArgument.Key, expectedKeys[i]); Assert.Equal(typeArgument.Value, expectedValues[i]); } } [Fact] public void FixedStringTypeArguments() { var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo("FixedString(42)"); Assert.Equal("FixedString", typeInfo.TypeName); Assert.Equal(0, typeInfo.GenericArgumentsCount); Assert.Equal(1, typeInfo.TypeArgumentsCount); Assert.Equal(42, typeInfo.GetTypeArgument(0)); } [Fact] public void NamedTupleArguments() { var typeNames = new[] { "UInt32", "Int64", "Nullable(String)", "Enum16('ok'=0, 'notOk'=8096)", "UInt16", "String", "Float64", "DateTime64(2, 'America/Los_Angeles')", "Nullable(Decimal(28, 4))" }; var itemNames = new KeyValuePair[] { new KeyValuePair("A","A "), new KeyValuePair("second_item"," second_item "), new KeyValuePair("B","`B` "), new KeyValuePair("_4","_4 "), new KeyValuePair("escaped `C` with \\` :)"," `escaped \\`C\\` with \\\\\\` :)` "), new KeyValuePair(" ([{some other name ","` ([{some other name ` "), new KeyValuePair("_O_O_"," _O_O_ "), new KeyValuePair("8"," \t`8`\t"), new KeyValuePair("OMEGA","\t OMEGA \t") }; Assert.Equal(typeNames.Length, itemNames.Length); for (int i = 1; i <= typeNames.Length; i++) { var tupleItems = Enumerable.Range(0, i).Select(j => itemNames[j].Value + typeNames[j]); var typeName = "Tuple(" + string.Join(',', tupleItems) + ')'; var typeInfo = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeName); ValidateTypeInfo(typeInfo, i); var typeInfoCopy = DefaultTypeInfoProvider.Instance.GetTypeInfo(typeInfo.ComplexTypeName); ValidateTypeInfo(typeInfoCopy, i); } void ValidateTypeInfo(IClickHouseColumnTypeInfo typeInfo, int expectedArgCount) { Assert.Equal("Tuple", typeInfo.TypeName); Assert.Equal(expectedArgCount, typeInfo.GenericArgumentsCount); Assert.Equal(expectedArgCount, typeInfo.TypeArgumentsCount); for (int i = 0; i < expectedArgCount; i++) { IClickHouseTypeInfo baseType = typeInfo.GetGenericArgument(i); Assert.Equal(typeNames[i], baseType.ComplexTypeName); var typeArgument = typeInfo.GetTypeArgument(i); var namedType = Assert.IsAssignableFrom>(typeArgument); Assert.Equal(itemNames[i].Key, namedType.Key); Assert.Equal(typeNames[i], namedType.Value.ComplexTypeName); } } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/CommonUtilsTests.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using Octonica.ClickHouseClient.Protocol; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class CommonUtilsTests { [Fact] public void GetColumnIndex() { var type = DefaultTypeInfoProvider.Instance.GetTypeInfo("Int32"); var colInfo = new List {new ColumnInfo("COL", type), new ColumnInfo("col", type), new ColumnInfo("another_col", type)}.AsReadOnly(); var idx = CommonUtils.GetColumnIndex(colInfo, "COL"); Assert.Equal(0, idx); idx = CommonUtils.GetColumnIndex(colInfo, "col"); Assert.Equal(1, idx); idx = CommonUtils.GetColumnIndex(colInfo, "AnOtHeR_cOl"); Assert.Equal(2, idx); Assert.Throws(() => CommonUtils.GetColumnIndex(colInfo, "there_is_no_column")); Assert.Throws(() => CommonUtils.GetColumnIndex(colInfo, "Col")); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/EncodingFixture.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System.Text; namespace Octonica.ClickHouseClient.Tests { internal class EncodingFixture { static EncodingFixture() { Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/IndexedCollectionTests.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Linq; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class IndexedCollectionTests { [Fact] public void Add() { var list = new[] { new KeyValuePair(1, "one"), new KeyValuePair(2, "two"), new KeyValuePair(3, "three"), new KeyValuePair(4, "four"), new KeyValuePair(5, "five"), new KeyValuePair(6, "six"), new KeyValuePair(7, "seven"), new KeyValuePair(8, "eight"), new KeyValuePair(9, "nine"), new KeyValuePair(0, "zero") }; var collection = new TestIndexedCollection(); Assert.Empty(collection); for (int i = 0; i < list.Length; i++) { collection.Clear(); Assert.Empty(collection); for (int j = 0; j <= i; j++) collection.Add(list[j]); Assert.Equal(i + 1, collection.Count); // Test IEnumerable Assert.Equal(list.Take(i + 1), collection); // Test indexer for (int j = 0; j <= i; j++) Assert.Equal(list[j], collection[j]); // Test key indexer for (int j = 0; j <= i; j++) Assert.Equal(list[j], collection[list[j].Key]); for (int j = 0; j < list.Length; j++) { bool shouldContain = j <= i; Assert.Equal(shouldContain, collection.Contains(list[j])); Assert.Equal(shouldContain, collection.ContainsKey(list[j].Key)); Assert.Equal(shouldContain, collection.TryGetValue(list[j].Key, out var actualValue)); if (shouldContain) Assert.Equal(list[j], actualValue); var expectedIndex = shouldContain ? j : -1; Assert.Equal(expectedIndex, collection.IndexOf(list[j])); Assert.Equal(expectedIndex, collection.IndexOf(list[j].Key)); } } } [Fact] public void Insert() { var list = new List> { new KeyValuePair("one", 1), new KeyValuePair("two", 2), new KeyValuePair("three", 3), new KeyValuePair("four", 4), new KeyValuePair("five", 5) }; var collection = new TestIndexedCollection(StringComparer.Ordinal); Assert.Empty(collection); for (int i = 0; i <= list.Count; i++) { collection.Clear(); Assert.Empty(collection); for (int j = list.Count - 1; j >= 0; j--) collection.Insert(0, list[j]); Assert.Equal(list.Count, collection.Count); var expectedList = list.ToList(); collection.Insert(i, new KeyValuePair("many", 42)); expectedList.Insert(i, new KeyValuePair("many", 42)); // Test IEnumerable Assert.Equal(expectedList, collection); // Test indexer for (int j = 0; j < expectedList.Count; j++) Assert.Equal(expectedList[j], collection[j]); // Test key indexer for (int j = 0; j < expectedList.Count; j++) Assert.Equal(expectedList[j], collection[expectedList[j].Key]); for (int j = 0; j < expectedList.Count; j++) { Assert.Contains(expectedList[j], collection); Assert.True(collection.ContainsKey(expectedList[j].Key)); Assert.True(collection.TryGetValue(expectedList[j].Key, out var actualValue)); Assert.Equal(expectedList[j], actualValue); Assert.Equal(j, collection.IndexOf(expectedList[j])); Assert.Equal(j, collection.IndexOf(expectedList[j].Key)); } } } [Theory] [InlineData("RemoveAt")] [InlineData("RemoveKey")] [InlineData("RemoveItem")] public void Remove(string removeMethod) { var list = new List> { new KeyValuePair("a", 2), new KeyValuePair("b", 4), new KeyValuePair("c", 8), new KeyValuePair("d", 16), new KeyValuePair("e", 32), new KeyValuePair("f", 64), }; var collection = new TestIndexedCollection(StringComparer.Ordinal); Assert.Empty(collection); for (int range = 1; range <= list.Count; range++) { for (int start = 0; start <= list.Count - range; start++) { collection.Clear(); Assert.Empty(collection); var expectedList = list.ToList(); for (int i = 0; i < list.Count; i++) collection.Add(list[i]); for (int i = range - 1; i >= 0; i--) { switch (removeMethod) { case "RemoveKey": var key = expectedList[start + i].Key; Assert.True(collection.Remove(key)); Assert.False(collection.Remove(key)); break; case "RemoveItem": var item = expectedList[start + i]; Assert.True(collection.Remove(item)); Assert.False(collection.Remove(item)); break; case "RemoveAt": collection.RemoveAt(start + i); break; default: Assert.True(false, $"Unknown method: {removeMethod}"); break; } expectedList.RemoveAt(start + i); } // Test IEnumerable Assert.Equal(expectedList, collection); // Test indexer for (int i = 0; i < expectedList.Count; i++) Assert.Equal(expectedList[i], collection[i]); // Test key indexer for (int i = 0; i < expectedList.Count; i++) Assert.Equal(expectedList[i], collection[expectedList[i].Key]); for (int i = 0; i < list.Count; i++) { bool shouldContain = i < start || i >= start + range; Assert.Equal(shouldContain, collection.Contains(list[i])); Assert.Equal(shouldContain, collection.ContainsKey(list[i].Key)); Assert.Equal(shouldContain, collection.TryGetValue(list[i].Key, out var actualValue)); if (shouldContain) Assert.Equal(list[i], actualValue); var expectedIndex = shouldContain ? (i < start ? i : i - range) : -1; Assert.Equal(expectedIndex, collection.IndexOf(list[i])); Assert.Equal(expectedIndex, collection.IndexOf(list[i].Key)); } } } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ListExtensionsTests.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Utils; using System; using System.Collections.Generic; using System.Linq; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ListExtensionsTests { [Fact] public void SliceArray() { var arrayAsList = (IList)Enumerable.Range(50, 200).ToArray(); TestSlice(Enumerable.Range(50, 100).ToList(), arrayAsList.Slice(0, 100)); TestSlice(Enumerable.Range(200, 50).ToList(), arrayAsList.Slice(150)); TestSlice(Enumerable.Range(100, 100).ToList(), arrayAsList.Slice(50, 100)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList, arrayAsList.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, arrayAsList.Slice(50).Slice(2, 7)); TestSlice(expectedList, arrayAsList.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(Enumerable.Range(52, 7).ToList(), arrayAsList.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => arrayAsList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => arrayAsList.Slice(2, 7).Slice(50)); Assert.Throws(() => arrayAsList.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => arrayAsList.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => arrayAsList.Slice(50).Slice(2, 7).Slice(0, 100)); var arrayAsRoList = (IReadOnlyList)arrayAsList; TestSlice(Enumerable.Range(50, 100).ToList(), arrayAsRoList.Slice(0, 100)); TestSlice(Enumerable.Range(200, 50).ToList(), arrayAsRoList.Slice(150)); TestSlice(Enumerable.Range(100, 100).ToList(), arrayAsRoList.Slice(50, 100)); TestSlice(expectedList, arrayAsRoList.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, arrayAsRoList.Slice(50).Slice(2, 7)); TestSlice(expectedList, arrayAsRoList.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(Enumerable.Range(52, 7).ToList(), arrayAsRoList.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => arrayAsRoList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => arrayAsRoList.Slice(2, 7).Slice(50)); Assert.Throws(() => arrayAsRoList.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => arrayAsRoList.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => arrayAsRoList.Slice(50).Slice(2, 7).Slice(0, 100)); } [Fact] public void SliceReadOnlyList() { var list = (IReadOnlyList)Enumerable.Range(50, 200).ToList(); TestSlice(Enumerable.Range(50, 100).ToList(), list.Slice(0, 100)); TestSlice(Enumerable.Range(200, 50).ToList(), list.Slice(150)); TestSlice(Enumerable.Range(100, 100).ToList(), list.Slice(50, 100)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList, list.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, list.Slice(50).Slice(2, 7)); TestSlice(expectedList, list.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(Enumerable.Range(52, 7).ToList(), list.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => list.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => list.Slice(2, 7).Slice(50)); Assert.Throws(() => list.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => list.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => list.Slice(50).Slice(2, 7).Slice(0, 100)); } [Fact] public void SliceList() { var list = new ListWrapper(Enumerable.Range(50, 200).ToList()); TestSlice(Enumerable.Range(50, 100).ToList(), list.Slice(0, 100)); TestSlice(Enumerable.Range(200, 50).ToList(), list.Slice(150)); TestSlice(Enumerable.Range(100, 100).ToList(), list.Slice(50, 100)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList, list.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, list.Slice(50).Slice(2, 7)); TestSlice(expectedList, list.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(Enumerable.Range(52, 7).ToList(), list.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => list.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => list.Slice(2, 7).Slice(50)); Assert.Throws(() => list.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => list.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => list.Slice(50).Slice(2, 7).Slice(0, 100)); } [Fact] public void MapArray() { var mappedArray = ((IReadOnlyList)Enumerable.Range(-50, 100).ToArray()).Map(v => v + 10); TestSlice(Enumerable.Range(-40, 100).ToList(), mappedArray); TestSlice(Enumerable.Range(-40, 100).Select(v => v * 2).ToList(), mappedArray.Map(v => v * 2)); mappedArray = ((IList)Enumerable.Range(-50, 100).ToArray()).Map(v => v - 10); TestSlice(Enumerable.Range(-60, 100).ToList(), mappedArray); TestSlice(Enumerable.Range(-60, 100).Select(v => v * 2).ToList(), mappedArray.Map(v => v * 2)); } [Fact] public void MapReadOnlyList() { var list = ((IReadOnlyList)Enumerable.Range(-50, 100).ToList()).Map(v => v + 10); TestSlice(Enumerable.Range(-40, 100).ToList(), list); TestSlice(Enumerable.Range(-40, 100).Select(v => v * 2).ToList(), list.Map(v => v * 2)); } [Fact] public void MapList() { var list = new ListWrapper(Enumerable.Range(-50, 100).ToList()).Map(v => v - 10); TestSlice(Enumerable.Range(-60, 100).ToList(), list); TestSlice(Enumerable.Range(-60, 100).Select(v => v * 2).ToList(), list.Map(v => v * 2)); } [Fact] public void SliceMapArray() { TestMappedArray(((IReadOnlyList)Enumerable.Range(50, 200).ToArray()).Map(v => v + 10)); TestMappedArray(((IList)Enumerable.Range(50, 200).ToArray()).Map(v => v + 10)); static void TestMappedArray(IReadOnlyList mappedArray) { TestSlice(Enumerable.Range(60, 200).ToList(), mappedArray); TestSlice(Enumerable.Range(60, 100).ToList(), mappedArray.Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedArray.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedArray.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), mappedArray.Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedArray.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedArray.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), mappedArray.Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedArray.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedArray.Slice(50, 100).Map(v => v - 10)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedArray.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedArray.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedArray.Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedArray.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedArray.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedArray.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedArray.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(Enumerable.Range(62, 7).ToList(), mappedArray.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => mappedArray.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(2, 7).Slice(50)); Assert.Throws(() => mappedArray.Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedArray.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => mappedArray.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedArray.Map(v => v - 10).Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedArray.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedArray.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedArray.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); } var array = (IReadOnlyList)Enumerable.Range(60, 200).ToArray(); TestSlice(Enumerable.Range(60, 100).ToList(), array.Slice(0, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(50, 100).ToList(), array.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), array.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), array.Slice(150).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(200, 50).ToList(), array.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), array.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), array.Slice(50, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(100, 100).ToList(), array.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), array.Slice(50, 100).Map(v => v - 10)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Map(v => v - 10).Map(v => v + 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Map(v => v - 10).Slice(0, 100).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Slice(0, 100).Map(v => v - 10).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), array.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10).Map(v => v + 10)); TestSlice(expectedList, array.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, array.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, array.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, array.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, array.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, array.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, array.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, array.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, array.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, array.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, array.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); Assert.Throws(() => array.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => array.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => array.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => array.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => array.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => array.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => array.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); } [Fact] public void SliceMapReadOnlyList() { var mappedRoList = ((IReadOnlyList)Enumerable.Range(50, 200).ToList()).Map(v => v + 10); TestSlice(Enumerable.Range(60, 200).ToList(), mappedRoList); TestSlice(Enumerable.Range(60, 100).ToList(), mappedRoList.Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedRoList.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedRoList.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), mappedRoList.Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedRoList.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedRoList.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), mappedRoList.Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedRoList.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedRoList.Slice(50, 100).Map(v => v - 10)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedRoList.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedRoList.Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedRoList.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedRoList.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(Enumerable.Range(62, 7).ToList(), mappedRoList.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => mappedRoList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(2, 7).Slice(50)); Assert.Throws(() => mappedRoList.Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedRoList.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => mappedRoList.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedRoList.Map(v => v - 10).Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedRoList.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedRoList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedRoList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); var roList = (IReadOnlyList)Enumerable.Range(60, 200).ToList().AsReadOnly(); TestSlice(Enumerable.Range(60, 100).ToList(), roList.Slice(0, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(50, 100).ToList(), roList.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), roList.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), roList.Slice(150).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(200, 50).ToList(), roList.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), roList.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), roList.Slice(50, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(100, 100).ToList(), roList.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), roList.Slice(50, 100).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Map(v => v + 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Slice(0, 100).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Map(v => v - 10).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10).Map(v => v + 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); Assert.Throws(() => roList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => roList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => roList.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => roList.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => roList.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => roList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => roList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); } [Fact] public void SliceMapList() { var mappedList = new ListWrapper(Enumerable.Range(50, 200).ToList()).Map(v => v + 10); TestSlice(Enumerable.Range(60, 200).ToList(), mappedList); TestSlice(Enumerable.Range(60, 100).ToList(), mappedList.Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedList.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), mappedList.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), mappedList.Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedList.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), mappedList.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), mappedList.Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedList.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), mappedList.Slice(50, 100).Map(v => v - 10)); var expectedList = Enumerable.Range(102, 7).ToList(); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedList.Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedList.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedList.Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedList.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), mappedList.Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedList.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, mappedList.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(Enumerable.Range(62, 7).ToList(), mappedList.Slice(0, 100).Slice(2, 7)); Assert.Throws(() => mappedList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(2, 7).Slice(50)); Assert.Throws(() => mappedList.Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedList.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => mappedList.Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedList.Map(v => v - 10).Slice(0, 100).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedList.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => mappedList.Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Map(v => v - 10).Slice(50).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => mappedList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); var roList = new ListWrapper(Enumerable.Range(60, 200).ToList()); TestSlice(Enumerable.Range(60, 100).ToList(), roList.Slice(0, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(50, 100).ToList(), roList.Map(v => v - 10).Slice(0, 100)); TestSlice(Enumerable.Range(50, 100).ToList(), roList.Slice(0, 100).Map(v => v - 10)); TestSlice(Enumerable.Range(210, 50).ToList(), roList.Slice(150).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(200, 50).ToList(), roList.Map(v => v - 10).Slice(150)); TestSlice(Enumerable.Range(200, 50).ToList(), roList.Slice(150).Map(v => v - 10)); TestSlice(Enumerable.Range(110, 100).ToList(), roList.Slice(50, 100).Map(v => v - 10).Map(v => v + 10)); TestSlice(Enumerable.Range(100, 100).ToList(), roList.Map(v => v - 10).Slice(50, 100)); TestSlice(Enumerable.Range(100, 100).ToList(), roList.Slice(50, 100).Map(v => v - 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Map(v => v + 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Slice(0, 100).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Map(v => v - 10).Map(v => v + 10).Slice(2, 7)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7).Map(v => v + 10)); TestSlice(expectedList.Select(v => v + 10).ToList(), roList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10).Map(v => v + 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(50).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Map(v => v - 10).Slice(0, 100).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(0, 100).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(0, 100).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(50).Slice(2, 7).Map(v => v - 10)); TestSlice(expectedList, roList.Map(v => v - 10).Slice(0, 100).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Map(v => v - 10).Slice(50).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Slice(50).Map(v => v - 10).Slice(2, 7)); TestSlice(expectedList, roList.Slice(0, 100).Slice(50).Slice(2, 7).Map(v => v - 10)); Assert.Throws(() => roList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => roList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => roList.Slice(2, 7).Map(v => v - 10).Slice(50)); Assert.Throws(() => roList.Slice(2, 7).Map(v => v - 10).Slice(0, 100)); Assert.Throws(() => roList.Slice(0, 100).Map(v => v - 10).Slice(2, 7).Slice(50)); Assert.Throws(() => roList.Slice(50).Map(v => v - 10).Slice(2, 7).Slice(0, 100)); Assert.Throws(() => roList.Slice(50).Slice(2, 7).Map(v => v - 10).Slice(0, 100)); } [Fact] public void ArrayCopyTo() { var array = Enumerable.Range(0, 99).ToArray(); TestCopyTo(array); } [Fact] public void ListCopyTo() { var list = Enumerable.Range(0, 99).ToList(); TestCopyTo(list); } private static void TestSlice(IReadOnlyList expected, IReadOnlyList slice) { Assert.Equal(expected.Count, slice.Count); var ex = Assert.ThrowsAny(() => slice[-1]); if(!(ex is IndexOutOfRangeException)) { var argumentEx = Assert.IsAssignableFrom(ex); Assert.Equal("index", argumentEx.ParamName); } ex = Assert.ThrowsAny(() => slice[expected.Count]); if (!(ex is IndexOutOfRangeException)) { var argumentEx = Assert.IsAssignableFrom(ex); Assert.Equal("index", argumentEx.ParamName); } // IEnumerable implementation Assert.Equal(expected, slice); // IReadOnlyList implementation for (int i = 0; i < expected.Count; i++) Assert.Equal(expected[i], slice[i]); TestCopyTo(slice); } private static void TestCopyTo(IReadOnlyList list) { var span = new Span(new T[list.Count + 16]); var length = list.CopyTo(span, list.Count); Assert.Equal(0, length); length = list.CopyTo(default, 0); Assert.Equal(0, length); Assert.Throws(() => list.CopyTo(new Span(new T[2], 1, 1), -1)); Assert.Throws(() => list.CopyTo(new Span(new T[1]), list.Count + 1)); length = list.CopyTo(span, 0); Assert.Equal(list.Count, length); for (int i = 0; i < length; i++) Assert.Equal(list[i], span[i]); var halfLength = list.Count / 2; length = list.CopyTo(span.Slice(halfLength, halfLength), halfLength / 2); Assert.Equal(Math.Min(list.Count - halfLength / 2, halfLength), length); for (int i = halfLength / 2, j = halfLength; i < halfLength / 2 + length; i++, j++) Assert.Equal(list[i], span[j]); } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/Octonica.ClickHouseClient.Tests.csproj ================================================  netcoreapp3.1;net6.0;net8.0 false enable all runtime; build; native; contentfiles; analyzers; buildtransitive all runtime; build; native; contentfiles; analyzers; buildtransitive Always PreserveNewest ================================================ FILE: src/Octonica.ClickHouseClient.Tests/ReadWriteBufferTests.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Buffers; using System.Linq; using Octonica.ClickHouseClient.Utils; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class ReadWriteBufferTests { [Fact] public void WriteBlocks() { var source = Enumerable.Range(0, 1000).Select(n => unchecked((byte) n)).ToArray(); for (int i = 0; i < 10; i++) { var buffer = new ReadWriteBuffer(7); var rnd = new Random(i); var position = 0; while (position < source.Length) { var blockSize = Math.Min(rnd.Next(1, 18), source.Length - position); var memory = buffer.GetMemory(blockSize); new ReadOnlySpan(source, position, blockSize).CopyTo(memory.Span); buffer.ConfirmWrite(blockSize); position += blockSize; } var empty = buffer.Read(); Assert.True(empty.IsEmpty); buffer.Flush(); var firstPart = buffer.Read(); var array = firstPart.Slice(0, 500).ToArray(); buffer.ConfirmRead(array.Length); Assert.Equal(source.Take(500), array); var secondPart = buffer.Read(); secondPart.CopyTo(array); buffer.ConfirmRead(array.Length); Assert.Equal(source.Skip(500), array); empty = buffer.Read(); Assert.True(empty.IsEmpty); } } [Fact] public void WriteExactThreeBlocks() { var source = Enumerable.Range(0, 7 * 3).Select(n => (byte) n).ToArray(); var buffer = new ReadWriteBuffer(7); for (int i = 0; i < 3; i++) { for (int j = 0; j < 3; j++) { var block = buffer.GetMemory(7); new Span(source, j * 7, 7).CopyTo(block.Span); buffer.ConfirmWrite(7); } buffer.Flush(); var result = buffer.Read().ToArray(); Assert.Equal(source, result); buffer.ConfirmRead(result.Length); var empty = buffer.Read(); Assert.True(empty.IsEmpty); } } [Fact] public void ReadWriteSequentialWithoutSegmentOverflow() { const int segmentSize = 13; var source = Enumerable.Range(0, 1000).Select(n => unchecked((byte) n)).ToArray(); var target = new byte[source.Length]; var buffer = new ReadWriteBuffer(segmentSize); for (int i = 0; i < 100; i++) { var rnd = new Random(i); Array.Clear(target, 0, target.Length); int sourcePosition = 0; int targetPosition = 0; int readWriteFactor = i % 3 + 2; while (sourcePosition != source.Length) { var read = rnd.Next(readWriteFactor) == 0; if (read) { var availableBytes = buffer.Read(); if (!availableBytes.IsEmpty) { var availableLen = Math.Min((int) availableBytes.Length, Math.Min(rnd.Next(segmentSize * 3) + 1, target.Length - targetPosition)); var targetSlice = new Span(target).Slice(targetPosition, availableLen); availableBytes.Slice(0, availableLen).CopyTo(targetSlice); targetPosition += availableLen; buffer.ConfirmRead(availableLen); continue; } } var len = Math.Min(source.Length - sourcePosition, rnd.Next(segmentSize * 3) + 1); while (len != 0) { var memory = buffer.GetMemory(); var currentLen = Math.Min(len, memory.Length); new ReadOnlySpan(source).Slice(sourcePosition, currentLen).CopyTo(memory.Span.Slice(0, currentLen)); buffer.ConfirmWrite(currentLen); sourcePosition += currentLen; len -= currentLen; } buffer.Flush(); } var lastBlock = buffer.Read(); Assert.Equal(target.Length - targetPosition, lastBlock.Length); lastBlock.CopyTo(new Span(target, targetPosition, target.Length - targetPosition)); Assert.Equal(source, target); buffer.ConfirmRead((int) lastBlock.Length); } } [Fact] public void ReadWriteSequentialWithSegmentOverflow() { const int segmentSize = 13; var source = Enumerable.Range(0, 1000).Select(n => unchecked((byte) n)).ToArray(); var target = new byte[source.Length]; var buffer = new ReadWriteBuffer(segmentSize); for (int i = 0; i < 100; i++) { var rnd = new Random(i); Array.Clear(target, 0, target.Length); int sourcePosition = 0; int targetPosition = 0; int readWriteFactor = i % 3 + 2; while (sourcePosition != source.Length) { var read = rnd.Next(readWriteFactor) == 0; if (read) { var availableBytes = buffer.Read(); if (!availableBytes.IsEmpty) { var availableLen = Math.Min((int) availableBytes.Length, Math.Min(rnd.Next(segmentSize * 3) + 1, target.Length - targetPosition)); var targetSlice = new Span(target).Slice(targetPosition, availableLen); availableBytes.Slice(0, availableLen).CopyTo(targetSlice); targetPosition += availableLen; buffer.ConfirmRead(availableLen); continue; } } var len = Math.Min(source.Length - sourcePosition, rnd.Next(segmentSize * 3) + 1); var memory = buffer.GetMemory(len); new ReadOnlySpan(source).Slice(sourcePosition, len).CopyTo(memory.Span.Slice(0, len)); buffer.ConfirmWrite(len); sourcePosition += len; buffer.Flush(); } var lastBlock = buffer.Read(); Assert.Equal(target.Length - targetPosition, lastBlock.Length); lastBlock.CopyTo(new Span(target, targetPosition, target.Length - targetPosition)); Assert.Equal(source, target); buffer.ConfirmRead((int) lastBlock.Length); } } [Fact] public void FlushDiscardWithoutSegmentOverflow() { const int segmentSize = 13; var source = Enumerable.Range(0, 1000).Select(n => unchecked((byte)n)).ToArray(); var target = new byte[source.Length]; var buffer = new ReadWriteBuffer(segmentSize); for (int i = 0; i < 100; i++) { var rnd = new Random(i); Array.Clear(target, 0, target.Length); int sourcePosition = 0; int targetPosition = 0; int discardPosition = 0; int readWriteFactor = i % 3 + 2; while (discardPosition != source.Length) { var read = rnd.Next(readWriteFactor) == 0; if (read) { var availableBytes = buffer.Read(); if (!availableBytes.IsEmpty) { var availableLen = Math.Min((int)availableBytes.Length, Math.Min(rnd.Next(segmentSize * 3) + 1, target.Length - targetPosition)); var targetSlice = new Span(target).Slice(targetPosition, availableLen); availableBytes.Slice(0, availableLen).CopyTo(targetSlice); targetPosition += availableLen; buffer.ConfirmRead(availableLen); continue; } } var len = Math.Min(source.Length - sourcePosition, rnd.Next(segmentSize * 3) + 1); while (len != 0) { var memory = buffer.GetMemory(); var currentLen = Math.Min(len, memory.Length); new ReadOnlySpan(source).Slice(sourcePosition, currentLen).CopyTo(memory.Span.Slice(0, currentLen)); buffer.ConfirmWrite(currentLen); sourcePosition += currentLen; len -= currentLen; } if (rnd.Next(4) == 0) { buffer.Flush(); discardPosition = sourcePosition; } else if (rnd.Next(2) == 0) { buffer.Discard(); sourcePosition = discardPosition; } } var lastBlock = buffer.Read(); Assert.Equal(target.Length - targetPosition, lastBlock.Length); lastBlock.CopyTo(new Span(target, targetPosition, target.Length - targetPosition)); Assert.Equal(source, target); buffer.ConfirmRead((int)lastBlock.Length); } } [Fact] public void FlushDiscardWithSegmentOverflow() { const int segmentSize = 13; var source = Enumerable.Range(0, 1000).Select(n => unchecked((byte)n)).ToArray(); var target = new byte[source.Length]; var buffer = new ReadWriteBuffer(segmentSize); for (int i = 0; i < 100; i++) { var rnd = new Random(i); Array.Clear(target, 0, target.Length); int sourcePosition = 0; int targetPosition = 0; int discardPosition = 0; int readWriteFactor = i % 3 + 2; while (discardPosition != source.Length) { var read = rnd.Next(readWriteFactor) == 0; if (read) { var availableBytes = buffer.Read(); if (!availableBytes.IsEmpty) { var availableLen = Math.Min((int)availableBytes.Length, Math.Min(rnd.Next(segmentSize * 3) + 1, target.Length - targetPosition)); var targetSlice = new Span(target).Slice(targetPosition, availableLen); availableBytes.Slice(0, availableLen).CopyTo(targetSlice); targetPosition += availableLen; buffer.ConfirmRead(availableLen); continue; } } var len = Math.Min(source.Length - sourcePosition, rnd.Next(segmentSize * 3) + 1); var memory = buffer.GetMemory(len); new ReadOnlySpan(source).Slice(sourcePosition, len).CopyTo(memory.Span.Slice(0, len)); buffer.ConfirmWrite(len); sourcePosition += len; if (rnd.Next(4) == 0) { buffer.Flush(); discardPosition = sourcePosition; } else if (rnd.Next(2) == 0) { buffer.Discard(); sourcePosition = discardPosition; } } var lastBlock = buffer.Read(); Assert.Equal(target.Length - targetPosition, lastBlock.Length); lastBlock.CopyTo(new Span(target, targetPosition, target.Length - targetPosition)); Assert.Equal(source, target); buffer.ConfirmRead((int)lastBlock.Length); } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/TestBox.cs ================================================ #region License Apache 2.0 /* Copyright 2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Tests { internal readonly struct TestBox { private readonly T _value; public TestBox(T value) { _value = value; } public T Unbox() => _value; } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/TestEnum.cs ================================================ #region License Apache 2.0 /* Copyright 2020 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion namespace Octonica.ClickHouseClient.Tests { internal enum TestEnum { None = 0, Value1 = 42, Value2 = -134, Value3 = 32000, Value4 = int.MaxValue } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/TestIndexedCollection.cs ================================================ #region License Apache 2.0 /* Copyright 2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using Octonica.ClickHouseClient.Utils; using System.Collections.Generic; namespace Octonica.ClickHouseClient.Tests { internal sealed class TestIndexedCollection : IndexedCollectionBase> where TKey : notnull { public TestIndexedCollection(IEqualityComparer? comparer = null) : base(comparer) { } protected override TKey GetKey(KeyValuePair item) { return item.Key; } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/TestListWrappers.cs ================================================ #region License Apache 2.0 /* Copyright 2020-2021 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace Octonica.ClickHouseClient.Tests { // Wrappers for List. Each wrapper implements only one of interfaces implemented by List. internal abstract class ListWrapperBase { public List List { get; } public int Count => List.Count; public bool IsReadOnly => ((IList) List).IsReadOnly; protected ListWrapperBase(List list) { List = list; } public IEnumerator GetEnumerator() { return List.GetEnumerator(); } public void Add(T item) { List.Add(item); } public void Clear() { List.Clear(); } public bool Contains(T item) { return List.Contains(item); } public void CopyTo(T[] array, int arrayIndex) { List.CopyTo(array, arrayIndex); } public bool Remove(T item) { return List.Remove(item); } public int IndexOf(T item) { return List.IndexOf(item); } public void Insert(int index, T item) { List.Insert(index, item); } public void RemoveAt(int index) { List.RemoveAt(index); } public T this[int index] { get => List[index]; set => List[index] = value; } } internal sealed class EnumerableListWrapper : ListWrapperBase, IEnumerable { public EnumerableListWrapper(List list) : base(list) { } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } } internal sealed class GenericEnumerableListWrapper : ListWrapperBase, IEnumerable { public GenericEnumerableListWrapper(List list) : base(list) { } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } } internal sealed class ListWrapper : ListWrapperBase, IList { public ListWrapper(List list) : base(list) { } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } } internal sealed class AsyncEnumerableListWrapper : IAsyncEnumerable { private readonly IReadOnlyList _list; public AsyncEnumerableListWrapper(IReadOnlyList list) { _list = list; } public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { return new Enumerator(_list); } private class Enumerator : IAsyncEnumerator { private readonly IReadOnlyList _list; private int _index = -1; public T Current => _list[_index]; public Enumerator(IReadOnlyList list) { _list = list; } public ValueTask DisposeAsync() { return default; } public ValueTask MoveNextAsync() { if (_index == _list.Count || ++_index == _list.Count) return new ValueTask(false); return new ValueTask(true); } } } internal sealed class Int32ToUInt32MappedListWrapper : ListWrapperBase, IReadOnlyList, IReadOnlyList { private readonly Func _map; public Int32ToUInt32MappedListWrapper(List list, Func map) : base(list) { _map = map; } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return this.Select(_map).GetEnumerator(); } uint IReadOnlyList.this[int index] => _map(this[index]); } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/TypeTests.cs ================================================ #region License Apache 2.0 /* Copyright 2019-2024 Octonica * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #endregion using System; using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Globalization; using System.Linq; using System.Net; using System.Numerics; using System.Text; using System.Threading; using System.Threading.Tasks; using Octonica.ClickHouseClient.Exceptions; using Octonica.ClickHouseClient.Types; using Octonica.ClickHouseClient.Utils; using Xunit; namespace Octonica.ClickHouseClient.Tests { public class TypeTests : ClickHouseTestsBase, IClassFixture { [Fact] public async Task ReadFixedStringScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT cast('1234йё' AS FixedString(10))"); var result = await cmd.ExecuteScalarAsync(); var resultBytes = Assert.IsType(result); Assert.Equal(10, resultBytes.Length); Assert.Equal(0, resultBytes[^1]); Assert.Equal(0, resultBytes[^2]); var resultString = Encoding.UTF8.GetString(resultBytes, 0, 8); Assert.Equal("1234йё", resultString); } [Fact] public async Task ReadFixedStringWithEncoding() { const string str = "аaбbсc"; var encoding = Encoding.GetEncoding("windows-1251"); await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT cast(convertCharset('{str}', 'UTF8', 'windows-1251') AS FixedString(10))"); var result = await cmd.ExecuteScalarAsync(); var resultBytes = Assert.IsType(result); Assert.Equal(10, resultBytes.Length); Assert.Equal(0, resultBytes[^1]); Assert.Equal(0, resultBytes[^2]); Assert.Equal(0, resultBytes[^3]); Assert.Equal(0, resultBytes[^4]); Assert.NotEqual(0, resultBytes[^5]); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureDataReader(new ClickHouseColumnSettings(encoding)); var success = await reader.ReadAsync(); Assert.True(success); var strResult = reader.GetString(0); Assert.Equal(str, strResult); success = await reader.ReadAsync(); Assert.False(success); var error = Assert.Throws(() => reader.ConfigureDataReader(new ClickHouseColumnSettings(Encoding.UTF8))); Assert.Equal(ClickHouseErrorCodes.DataReaderError, error.ErrorCode); } [Fact] public async Task ReadNullableFixedStringAsArray() { await WithTemporaryTable("fixed_string_as_array", "id Int32, str Nullable(FixedString(32))", Test); async Task Test(ClickHouseConnection connection, string tableName) { var stringValues = new string?[] { null, "фываasdf", "abcdef", "ghijkl", "null", "пролджэzcvgb", "ячсмить" }; var byteValues = stringValues.Select(v => v == null ? null : Encoding.UTF8.GetBytes(v)).ToArray(); await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, str) VALUES", CancellationToken.None)) { writer.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.UTF8)); await writer.WriteTableAsync(new object[] { Enumerable.Range(0, 10_000), stringValues }, stringValues.Length, CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, str FROM {tableName} ORDER BY id"); int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn(1, new ClickHouseColumnSettings(Encoding.UTF8)); char[] charBuffer = new char[stringValues.Select(v => v?.Length ?? 0).Max() + 7]; byte[] byteBuffer = new byte[32 + 3]; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); Assert.Equal(count, id); var expectedStr = stringValues[id]; var valueAsCharArray = reader.GetFieldValue(1, null); Assert.Equal(expectedStr, valueAsCharArray == null ? null : new string(valueAsCharArray)); var valueAsByteArray = reader.GetFieldValue(1, null); byte[]? expectedByteArray = byteValues[id] == null ? null : new byte[32]; byteValues[id]?.CopyTo((Memory)expectedByteArray); Assert.Equal(expectedByteArray, valueAsByteArray); if (expectedStr == null) { Assert.True(reader.IsDBNull(1)); } else { var len = (int)reader.GetChars(1, 0, charBuffer, 0, charBuffer.Length); Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(0, len))); len = (int)reader.GetBytes(1, 0, byteBuffer, 0, byteBuffer.Length); Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Take(len)); len = 0; while (len < charBuffer.Length - 7) { var size = Math.Min(3, charBuffer.Length - len - 7); var currentLen = (int)reader.GetChars(1, len, charBuffer, len + 7, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(7, len))); len = 0; while (len < byteBuffer.Length - 3) { var size = Math.Min(3, byteBuffer.Length - len - 3); var currentLen = (int)reader.GetBytes(1, len, byteBuffer, len + 3, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Skip(3).Take(len)); } ++count; } Assert.Equal(stringValues.Length, count); } } [Fact] public async Task ReadStringWithEncoding() { const string str = "АБВГДЕ"; var encoding = Encoding.GetEncoding("windows-1251"); await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT convertCharset('{str}', 'UTF8', 'windows-1251') AS c"); await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureDataReader(new ClickHouseColumnSettings(encoding)); var success = await reader.ReadAsync(); Assert.True(success); var error = Assert.Throws(() => reader.ConfigureColumn("c", new ClickHouseColumnSettings(Encoding.UTF8))); Assert.Equal(ClickHouseErrorCodes.DataReaderError, error.ErrorCode); var strResult = reader.GetString(0); Assert.Equal(str, strResult); success = await reader.ReadAsync(); Assert.False(success); } [Fact] public async Task ReadStringWithEncodingScalar() { const string str = "АБВГДЕ"; var encoding = Encoding.GetEncoding("windows-1251"); await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT convertCharset('{str}', 'UTF8', 'windows-1251') AS c"); var strResult = await cmd.ExecuteScalarAsync(new ClickHouseColumnSettings(encoding)); Assert.Equal(str, strResult); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadStringParameterScalar(ClickHouseParameterMode parameterMode) { var strings = new[] {null, "", "abcde", "fghi", "jklm", "ab\tc", "'abc'", "new\r\nline", "\\"}; var byteArray = Encoding.UTF8.GetBytes(strings[4]!); var byteMemory = Encoding.UTF8.GetBytes(strings[5]!).AsMemory(); var byteRoMemory = (ReadOnlyMemory) Encoding.UTF8.GetBytes(strings[6]!).AsMemory(); var charMemory = strings[7]!.ToCharArray().AsMemory(); var charArray = strings[8]!.ToCharArray(); var values = new object?[] {strings[0], strings[1], strings[2], strings[3].AsMemory(), byteArray, byteMemory, byteRoMemory, charMemory, charArray}; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {val}"); var param = cmd.Parameters.AddWithValue("val", "some_value", DbType.String); for (var i = 0; i < strings.Length; i++) { param.Value = values[i]; var result = await cmd.ExecuteScalarAsync(CancellationToken.None); if (values[i] == null) Assert.Equal(result, DBNull.Value); else Assert.Equal(strings[i], Assert.IsType(result)); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadStringArrayParameterScalar(ClickHouseParameterMode parameterMode) { var arr = new[] { "abc", "\r\n\t'\\", "\\", null, "", "ЮНИКОД", "'quoted'" }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {val} v, toTypeName(v)"); var param = cmd.Parameters.AddWithValue("val", arr); await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var typeName = reader.GetString(1); Assert.Equal("Array(Nullable(String))", typeName); var value = reader.GetValue(0); var valueArray = Assert.IsType(value); Assert.Equal(arr.Length, valueArray.Length); Assert.Equal(arr, valueArray); Assert.False(await reader.ReadAsync()); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadMultidimensionalStringArrayParameterScalar(ClickHouseParameterMode parameterMode) { var arr = new List() { new[,] { {"ab", "cd"}, {"ef", null} }, new[,] { {null, "foo", "\\"}, {"bar\tbaz", "new\r\nline", "rock'n'roll"} } }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {val} v, toTypeName(v)"); var param = cmd.Parameters.AddWithValue("val", arr); await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var typeName = reader.GetString(1); Assert.Equal("Array(Array(Array(Nullable(String))))", typeName); var value = reader.GetValue(0); var valueArray = Assert.IsType(value); Assert.Equal(arr.Count, valueArray.Length); for (int i = 0; i < valueArray.Length; i++) { var expected = arr[i]; var actualArr2 = valueArray[i]; var expectedLengthI = expected.GetLength(0); var expectedLengthJ = expected.GetLength(1); Assert.Equal(expectedLengthI, actualArr2.Length); for (int j = 0; j < expectedLengthI; j++) { var actualArr = actualArr2[j]; Assert.Equal(expectedLengthJ, actualArr.Length); for (int k = 0; k < expectedLengthJ; k++) Assert.Equal(expected[j, k], actualArr[k]); } } Assert.False(await reader.ReadAsync()); } [Fact] public async Task ReadGuidScalar() { var guidValue = new Guid("74D47928-2423-4FE2-AD45-82E296BF6058"); await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT cast('{guidValue:D}' AS UUID)"); var result = await cmd.ExecuteScalarAsync(); var resultGuid = Assert.IsType(result); Assert.Equal(guidValue, resultGuid); } [Fact] public async Task ReadDecimal128Scalar() { var testData = new[] {decimal.Zero, decimal.One, decimal.MinusOne, decimal.MinValue / 100, decimal.MaxValue / 100, decimal.One / 100, decimal.MinusOne / 100}; await using var connection = await OpenConnectionAsync(); foreach (var testValue in testData) { await using var cmd = connection.CreateCommand($"SELECT cast('{testValue.ToString(CultureInfo.InvariantCulture)}' AS Decimal128(2))"); var result = await cmd.ExecuteScalarAsync(); var resultDecimal = Assert.IsType(result); Assert.Equal(testValue, resultDecimal); } await using (var cmd2 = connection.CreateCommand("SELECT cast('-108.4815162342' AS Decimal128(35))")) { var result2 = await cmd2.ExecuteScalarAsync(); Assert.Equal(-108.4815162342m, result2); } await using (var cmd2 = connection.CreateCommand("SELECT cast('-999.9999999999999999999999999' AS Decimal128(35))")) { var result2 = await cmd2.ExecuteScalarAsync(); Assert.Equal(-999.9999999999999999999999999m, result2); } } [Fact] public async Task ReadDecimal64Scalar() { var testData = new[] { decimal.Zero, decimal.One, decimal.MinusOne, 999_999_999_999_999.999m, -999_999_999_999_999.999m, decimal.One / 1000, decimal.MinusOne / 1000 }; await using var connection = await OpenConnectionAsync(); foreach (var testValue in testData) { await using var cmd = connection.CreateCommand($"SELECT cast('{testValue.ToString(CultureInfo.InvariantCulture)}' AS Decimal64(3))"); var result = await cmd.ExecuteScalarAsync(); var resultDecimal = Assert.IsType(result); Assert.Equal(testValue, resultDecimal); } } [Fact] public async Task ReadDecimal32Scalar() { var testData = new[] { decimal.Zero, decimal.One, decimal.MinusOne, 9.9999999m, -9.9999999m, decimal.One / 100_000_000, decimal.MinusOne / 100_000_000 }; await using var connection = await OpenConnectionAsync(); foreach (var testValue in testData) { await using var cmd = connection.CreateCommand($"SELECT cast('{testValue.ToString(CultureInfo.InvariantCulture)}' AS Decimal32(8))"); var result = await cmd.ExecuteScalarAsync(); var resultDecimal = Assert.IsType(result); Assert.Equal(testValue, resultDecimal); } } [Fact] public async Task ReadDateTimeScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT cast('2015-04-21 14:59:44' AS DateTime)"); var result = await cmd.ExecuteScalarAsync(); var resultDateTime = Assert.IsType(result); Assert.Equal(new DateTime(2015, 4, 21, 14, 59, 44), resultDateTime.DateTime); } [Fact] public async Task ReadDateTimeWithTimezoneScalar() { await using var connection = await OpenConnectionAsync(); var tzName = TimeZoneHelper.GetTimeZoneId(TimeZoneInfo.Local); await using var cmd = connection.CreateCommand($"SELECT toDateTime('2015-04-21 14:59:44', '{tzName}')"); var result = await cmd.ExecuteScalarAsync(); var resultDateTime = Assert.IsType(result); Assert.Equal(new DateTime(2015, 4, 21, 14, 59, 44), resultDateTime); } [Theory] [InlineData("2015-04-21 14:59:44.12345", 3, 2015, 4, 21, 14, 59, 44, 123)] [InlineData("1908-09-10 11:12:13.141516", 6, 1908, 9, 10, 11, 12, 13, 141.516)] [InlineData("2112-01-15 23:01:59.99999", 1, 2112, 1, 15, 23, 1, 59, 900)] [InlineData("1970-01-01", 3, 1970, 1, 1, 0, 0, 0, 0)] [InlineData("1931-03-05 07:09:23", 0, 1931, 3, 5, 7, 9, 23, 0)] [InlineData("1900-01-01", 3, 1900, 1, 1, 0, 0, 0, 0)] // Min value [InlineData("2299-12-31 23:59:59.999999", 6, 2299, 12, 31, 23, 59, 59, 999.999)] // Max value public async Task ReadDateTime64Scalar(string dateText, int scale, int year, int month, int day, int hour, int minute, int second, double milliseconds) { await using var connection = await OpenConnectionAsync(); string queryStr = $"SELECT cast('{dateText}' AS DateTime64({scale}, 'Etc/UTC'))"; await using var cmd = connection.CreateCommand(queryStr); var result = await cmd.ExecuteScalarAsync(); var resultDateTime = Assert.IsType(result); Assert.Equal(new DateTime(year, month, day, hour, minute, second).Add(TimeSpan.FromMilliseconds(milliseconds)), resultDateTime.DateTime); } [Fact] public async Task ReadDateTime64WithTimezoneScalar() { await using var connection = await OpenConnectionAsync(); var tzName = TimeZoneHelper.GetTimeZoneId(TimeZoneInfo.Local); await using var cmd = connection.CreateCommand($"SELECT cast('2015-04-21 14:59:44.123456789' AS DateTime64(9,'{tzName}'))"); var result = await cmd.ExecuteScalarAsync(); var resultDateTime = Assert.IsType(result); Assert.Equal(new DateTime(2015, 4, 21, 14, 59, 44).Add(TimeSpan.FromMilliseconds(123.4567)), resultDateTime); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateTime64ParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); var timeZone = connection.GetServerTimeZone(); var unixEpochOffset = timeZone.GetUtcOffset(DateTime.UnixEpoch); var minDate = new DateTime(1900, 1, 1); var maxDate = new DateTime(2299, 12, 31, 23, 59, 59, 999); var values = new[] { default, DateTime.UnixEpoch.Add(TimeSpan.FromMilliseconds(1111.1111) + unixEpochOffset), new DateTime(1966, 6, 6, 7, 8, 9).Add(TimeSpan.FromMilliseconds(987.654)), new DateTime(1931, 3, 5, 7, 9, 23).Add(TimeSpan.FromMilliseconds(123.45)), new DateTime(1984, 4, 21, 14, 59, 44).Add(TimeSpan.FromMilliseconds(123.4567)), new DateTime(2032, 3, 18, 12, 0, 59).Add(TimeSpan.FromMilliseconds(987.6543)), new DateTime(1970, 1, 1), new DateTime(1970, 1, 1) + unixEpochOffset, minDate + timeZone.GetUtcOffset(minDate), maxDate + timeZone.GetUtcOffset(maxDate) }; await using var cmd = connection.CreateCommand("SELECT {v}"); var parameter = new ClickHouseParameter("v") {ClickHouseDbType = ClickHouseDbType.DateTime64}; cmd.Parameters.Add(parameter); for (int precision = 0; precision < 10; precision++) { parameter.Precision = (byte) precision; var div = TimeSpan.TicksPerSecond / (long) Math.Pow(10, precision); if (div == 0) div = -(long) Math.Pow(10, precision) / TimeSpan.TicksPerSecond; foreach (var value in values) { if (value.Year >= maxDate.Year && precision >= 9) { // The value is too large for writing continue; } parameter.Value = value; var result = await cmd.ExecuteScalarAsync(); var resultDateTime = Assert.IsType(result); DateTime expectedValue; if (value == default) expectedValue = DateTime.UnixEpoch + unixEpochOffset; else if (div > 0) expectedValue = new DateTime(value.Ticks / div * div); else expectedValue = value; Assert.Equal(expectedValue, resultDateTime.DateTime); } } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateTimeParameterWithTimezoneScalar(ClickHouseParameterMode parameterMode) { var valueShort = new DateTime(2014, 7, 5, 12, 13, 14); var value = valueShort.Add(TimeSpan.FromMilliseconds(123.4567)); const string targetTzCode = "Asia/Magadan"; var targetTz = TimeZoneHelper.GetTimeZoneInfo(targetTzCode); await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand($"SELECT toTimeZone({{d}}, '{targetTzCode}')"); var parameter = new ClickHouseParameter("d") {Value = value, TimeZone = TimeZoneHelper.GetTimeZoneInfo("Pacific/Niue"), Precision = 4}; cmd.Parameters.Add(parameter); var deltaOffset = targetTz.GetUtcOffset(valueShort) - parameter.TimeZone.GetUtcOffset(valueShort); object? resultObj; DateTimeOffset result; foreach (var parameterType in new ClickHouseDbType?[] {null, ClickHouseDbType.DateTime, ClickHouseDbType.DateTimeOffset}) { if (parameterType != null) parameter.ClickHouseDbType = parameterType.Value; resultObj = await cmd.ExecuteScalarAsync(); result = Assert.IsType(resultObj); Assert.Equal(valueShort + deltaOffset, result.DateTime); Assert.Equal(targetTz.GetUtcOffset(result), result.Offset); } parameter.ClickHouseDbType = ClickHouseDbType.DateTime2; resultObj = await cmd.ExecuteScalarAsync(); result = Assert.IsType(resultObj); Assert.Equal(value + deltaOffset, result.DateTime); Assert.Equal(targetTz.GetUtcOffset(result), result.Offset); parameter.ClickHouseDbType = ClickHouseDbType.DateTime64; resultObj = await cmd.ExecuteScalarAsync(); result = Assert.IsType(resultObj); Assert.Equal(valueShort + deltaOffset + TimeSpan.FromMilliseconds(123.4), result.DateTime); Assert.Equal(targetTz.GetUtcOffset(result), result.Offset); parameter.ResetDbType(); resultObj = await cmd.ExecuteScalarAsync(); result = Assert.IsType(resultObj); deltaOffset = targetTz.GetUtcOffset(valueShort) - connection.GetServerTimeZone().GetUtcOffset(valueShort); Assert.Equal(valueShort + deltaOffset, result.DateTime); } [Fact] public async Task ReadFloatScalar() { await using var connection = await OpenConnectionAsync(); var expectedValue = 1234567890.125f; await using var cmd = connection.CreateCommand($"SELECT CAST('{expectedValue:#.#}' AS Float32)"); var result = await cmd.ExecuteScalarAsync(); var resultFloat = Assert.IsType(result); Assert.Equal(expectedValue, resultFloat); } [Fact] public async Task ReadDoubleScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT -123456789109876.125"); var result = await cmd.ExecuteScalarAsync(); var resultDouble = Assert.IsType(result); Assert.Equal(-123456789109876.125, resultDouble); } [Fact] public async Task ReadNothingScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT NULL"); var result = await cmd.ExecuteScalarAsync(); Assert.IsType(result); } [Fact] public async Task ReadEmptyArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT []"); var result = await cmd.ExecuteScalarAsync(); var objResult = Assert.IsType(result); Assert.Empty(objResult); } [Fact] public async Task ReadByteArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT [4, 8, 15, 16, 23, 42]"); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(new byte[] {4, 8, 15, 16, 23, 42}, result); } [Fact] public async Task ReadNullableByteArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT [4, NULL, 8, NULL, 15, NULL, 16, NULL, 23, NULL, 42]"); var result = await cmd.ExecuteScalarAsync(); var resultArr = Assert.IsType(result); Assert.Equal(new byte?[] { 4, null, 8, null, 15, null, 16, null, 23, null, 42 }, resultArr); } [Fact] public async Task ReadArrayOfArraysOfArraysScalar() { const string query = @"SELECT [ [ [1], [], [2, NULL] ], [ [3] ] ]"; await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand(query); var result = await cmd.ExecuteScalarAsync(); var resultArr = Assert.IsType(result); Assert.NotNull(resultArr); Assert.Equal(2, resultArr.Length); Assert.NotNull(resultArr[0]); Assert.Equal(3, resultArr[0].Length); Assert.Equal(new byte?[] {1}, resultArr[0][0]); Assert.Equal(new byte?[0], resultArr[0][1]); Assert.Equal(new byte?[] {2, null}, resultArr[0][2]); Assert.NotNull(resultArr[1]); Assert.Equal(1, resultArr[1].Length); Assert.Equal(new byte?[] {3}, resultArr[1][0]); } [Fact] public async Task ReadNullableByteArrayAsUInt64ArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT [4, NULL, 8, NULL, 15, NULL, 16, NULL, 23, NULL, 42]"); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(new ulong?[] { 4, null, 8, null, 15, null, 16, null, 23, null, 42 }, result); } [Fact] public async Task ReadNullableStringArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT ['All', NULL, 'your', NULL, 'base', NULL, 'are', NULL, 'belong', NULL, 'to', NULL, 'us!']"); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(new[] {"All", null, "your", null, "base", null, "are", null, "belong", null, "to", null, "us!"}, result); } [Fact] public async Task ReadNullableNothingArrayScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT [NULL,NULL,NULL]"); var result = await cmd.ExecuteScalarAsync(); var resultArr = Assert.IsType(result); Assert.Equal(new object[3], resultArr); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadArrayParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {p}"); var expectedResult = new[] {4, 8, 15, 16, 23, 42}; var param = cmd.Parameters.AddWithValue("p", expectedResult); var result = await cmd.ExecuteScalarAsync(); var intResult = Assert.IsType(result); Assert.Equal(expectedResult, intResult); param.IsArray = true; param.DbType = DbType.Decimal; result = await cmd.ExecuteScalarAsync(); var decResult = Assert.IsType(result); Assert.Equal(expectedResult.Select(v => (decimal) v), decResult); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadArrayOfArraysParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {p}"); var expectedResult = new List {new uint[] {4, 8}, new uint[] {15, 16, 23}, new uint[] {42}}; var param = cmd.Parameters.AddWithValue("p", expectedResult); var result = await cmd.ExecuteScalarAsync(); var intResult = Assert.IsType(result); Assert.Equal(expectedResult.Count, intResult.Length); for (int i = 0; i < expectedResult.Count; i++) Assert.Equal(expectedResult[i], intResult[i]); param.ArrayRank = 2; param.DbType = DbType.UInt64; param.IsNullable = true; result = await cmd.ExecuteScalarAsync(); var decResult = Assert.IsType(result); Assert.Equal(expectedResult.Count, decResult.Length); for (int i = 0; i < decResult.Length; i++) Assert.Equal(expectedResult[i].Select(v => (ulong?) v), decResult[i]); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadMultidimensionalArrayParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {p}"); var expectedResult = new List {new[,] {{(int?) 4, 8}, {15, 16}, {23, 42}}, new[,] {{1}, {(int?) null}, {3}}, new[,] {{(int?) -4, -8, -15}, {-16, -23, -42}}}; var param = cmd.Parameters.AddWithValue("p", expectedResult); var result = await cmd.ExecuteScalarAsync(); var intResult = Assert.IsType(result); Assert.Equal(expectedResult.Count, intResult.Length); for (int i = 0; i < expectedResult.Count; i++) { var expectedLengthI = expectedResult[i].GetLength(0); var expectedLengthJ = expectedResult[i].GetLength(1); Assert.Equal(expectedLengthI, intResult[i].Length); for (int j = 0; j < expectedLengthI; j++) { Assert.Equal(expectedLengthJ, intResult[i][j].Length); for (int k = 0; k < expectedLengthJ; k++) Assert.Equal(expectedResult[i][j, k], intResult[i][j][k]); } } param.ArrayRank = 3; param.DbType = DbType.Decimal; result = await cmd.ExecuteScalarAsync(); var decResult = Assert.IsType(result); Assert.Equal(expectedResult.Count, decResult.Length); for (int i = 0; i < expectedResult.Count; i++) { var expectedLengthI = expectedResult[i].GetLength(0); var expectedLengthJ = expectedResult[i].GetLength(1); Assert.Equal(expectedLengthI, decResult[i].Length); for (int j = 0; j < expectedLengthI; j++) { Assert.Equal(expectedLengthJ, decResult[i][j].Length); for (int k = 0; k < expectedLengthJ; k++) Assert.Equal(expectedResult[i][j, k], decResult[i][j][k]); } } } [Fact] public async Task ReadEnumColumn() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand(@"SELECT CAST(T.col AS Enum(''=0, '\\e\s\c\\a\p\\e'=1, '\'val\''=2, '\r\n\t\d\\\r\n'=3,'\a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x20\y\z'=4)) AS enumVal, toString(enumVal) AS strVal FROM (SELECT 0 AS col UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4) AS T"); await using var reader = await cmd.ExecuteReaderAsync(); var columnType = reader.GetFieldTypeInfo(0); var typeNames = new Dictionary(); for(int i=0; i>(obj); typeNames.Add(pair.Value, pair.Key); } int bitmap = 0; while (await reader.ReadAsync()) { var value = reader.GetFieldValue(0); var defaultValue = reader.GetValue(0); var strValue = Assert.IsType(defaultValue); var expectedStrValue = reader.GetFieldValue(1); Assert.Equal(expectedStrValue, strValue); switch (value) { case 0: Assert.Equal(string.Empty, strValue); break; case 1: Assert.Equal(@"\e\s\c\a\p\e", strValue); break; case 2: Assert.Equal("'val'", strValue); break; case 3: Assert.Equal("\r\n\t\\d\\\r\n", strValue); break; case 4: Assert.Equal("\a\b\\c\\d\u001b\f\\g\\h\\i\\j\\k\\l\\m\n\\o\\p\\q\r\\s\t\\u\v\\w\x20\\y\\z", strValue); break; default: Assert.True(false, $"Unexpected value: {value}."); break; } Assert.Equal(strValue, typeNames[value]); bitmap ^= 1 << value; } Assert.Equal(31, bitmap); } [Fact] public async Task ReadEnumScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT CAST(42 AS Enum('' = 0, 'b' = -129, 'Hello, world! :)' = 42))"); var result = await cmd.ExecuteScalarAsync(); Assert.Equal("Hello, world! :)", result); } [Fact] public async Task ReadClrEnumScalar() { await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand("SELECT CAST(42 AS Enum('' = 0, 'b' = -129, 'Hello, world! :)' = 42))"); var settings = new ClickHouseColumnSettings(new ClickHouseEnumConverter()); var result = await cmd.ExecuteScalarAsync(settings); Assert.Equal(TestEnum.Value1, result); } [Fact] public async Task ReadInt32ArrayColumn() { int?[]?[] expected = new int?[10][]; var queryBuilder = new StringBuilder("SELECT T.* FROM (").AppendLine(); for (int i = 0, k = 0; i < expected.Length; i++) { if (i > 0) queryBuilder.AppendLine().Append("UNION ALL "); queryBuilder.Append("SELECT ").Append(i).Append(" AS num, ["); var expectedArray = expected[i] = new int?[i + 1]; for (int j = 0; j < expectedArray.Length; j++, k++) { if (j > 0) queryBuilder.Append(", "); if ((k % 3 == 0) == (k % 5 == 0)) { queryBuilder.Append("CAST(").Append(k).Append(" AS Nullable(Int32))"); expectedArray[j] = k; } else { queryBuilder.Append("CAST(NULL AS Nullable(Int32))"); } } queryBuilder.Append("] AS arr"); } var queryString = queryBuilder.AppendLine(") AS T").Append("ORDER BY T.num DESC").ToString(); await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(queryString); await using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { var num = reader.GetInt32(0); var expectedArray = expected[num]; Assert.NotNull(expectedArray); var value = reader.GetValue(1); var array = Assert.IsType(value); Assert.Equal(expectedArray, array); expected[num] = null; } Assert.All(expected, Assert.Null); } [Fact] public async Task SkipInt32ArrayColumn() { int?[]?[] expected = new int?[10][]; var queryBuilder = new StringBuilder("SELECT T.* FROM (").AppendLine(); for (int i = 0, k = 0; i < expected.Length; i++) { if (i > 0) queryBuilder.AppendLine().Append("UNION ALL "); queryBuilder.Append("SELECT ").Append(i).Append(" AS num, ["); var expectedArray = expected[i] = new int?[i + 1]; for (int j = 0; j < expectedArray.Length; j++, k++) { if (j > 0) queryBuilder.Append(", "); if ((k % 3 == 0) == (k % 5 == 0)) { queryBuilder.Append("CAST(").Append(k).Append(" AS Nullable(Int32))"); expectedArray[j] = k; } else { queryBuilder.Append("CAST(NULL AS Nullable(Int32))"); } } queryBuilder.Append("] AS arr"); } var queryString = queryBuilder.AppendLine(") AS T").Append("ORDER BY T.num DESC").ToString(); await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(queryString); await cmd.ExecuteNonQueryAsync(); } [Fact] public async Task ReadTuplesWithDifferentLength() { var tupleSb = new StringBuilder("tuple("); var querySb = new StringBuilder("SELECT").AppendLine(); for (int i = 0; i < 15; i++) { if (i > 0) { tupleSb.Append(", "); querySb.AppendLine(", "); } tupleSb.Append("CAST(").Append(i + 1).Append(" AS Int32)"); querySb.Append(tupleSb, 0, tupleSb.Length).Append($") AS t{i}"); } var queryString = querySb.ToString(); await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(queryString); await using var reader = await cmd.ExecuteReaderAsync(); var success = await reader.ReadAsync(); Assert.True(success); var expected = new object[] { Tuple.Create(1), Tuple.Create(1, 2), Tuple.Create(1, 2, 3), Tuple.Create(1, 2, 3, 4), Tuple.Create(1, 2, 3, 4, 5), Tuple.Create(1, 2, 3, 4, 5, 6), Tuple.Create(1, 2, 3, 4, 5, 6, 7), Tuple.Create(1, 2, 3, 4, 5, 6, 7, 8), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9)), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10)), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10, 11)), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10, 11, 12)), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10, 11, 12, 13)), new Tuple>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10, 11, 12, 13, 14)), new Tuple>>(1, 2, 3, 4, 5, 6, 7, Tuple.Create(8, 9, 10, 11, 12, 13, 14, 15)) }; for (int i = 0; i < 15; i++) { var columnType = reader.GetFieldType(i); Assert.Equal(expected[i].GetType(), columnType); var columnValue = reader.GetValue(i); if (i == 12) { var reinterpretedValue = reader.GetFieldValue>>(i); var expectedValue = new Tuple>( 1, 2, 3, 4, 5, 6, 7, Tuple.Create((long) 8, (long?) 9, 10, (int?) 11, 12, (int?) 13)); Assert.Equal(expectedValue, reinterpretedValue); } Assert.Equal(expected[i].GetType(), columnValue.GetType()); Assert.Equal(columnValue, expected[i]); } Assert.False(await reader.ReadAsync()); } [Fact] public async Task ReadValueTuplesWithDifferentLength() { var tupleSb = new StringBuilder("tuple("); var querySb = new StringBuilder("SELECT").AppendLine(); for (int i = 0; i < 15; i++) { if (i > 0) { tupleSb.Append(", "); querySb.AppendLine(", "); } tupleSb.Append("CAST(").Append(i + 1).Append(" AS Int32)"); querySb.Append(tupleSb, 0, tupleSb.Length).Append($") AS t{i}"); } var queryString = querySb.ToString(); await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(queryString); await using var reader = await cmd.ExecuteReaderAsync(); var success = await reader.ReadAsync(); Assert.True(success); Assert.Equal(15, reader.FieldCount); for (int i = 0; i < reader.FieldCount; i++) { switch (i) { case 0: AssertEqual(reader, i, new ValueTuple(1)); break; case 1: AssertEqual(reader, i, (1, (int?) 2)); break; case 2: AssertEqual(reader, i, (1, 2, 3)); break; case 3: AssertEqual(reader, i, (1, 2, 3, 4)); break; case 4: AssertEqual(reader, i, (1, 2, 3, 4, 5)); break; case 5: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6)); break; case 6: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7)); break; case 7: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, (long) 8)); break; case 8: AssertEqual(reader, i, (1, 2, 3, 4, (int?) 5, 6, 7, (long?) 8, 9)); break; case 9: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); break; case 10: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)); break; case 11: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)); break; case 12: AssertEqual<(int?, long, long?, int, int?, int, int?, long, long?, int, int?, int, int?)>(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); break; case 13: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); break; case 14: AssertEqual(reader, i, (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, (long?) 15)); break; default: Assert.True(i >= 0 && i < 15, "Too many columns."); break; } } Assert.False(await reader.ReadAsync()); static void AssertEqual(DbDataReader reader, int ordinal, T expectedValue) { var value = reader.GetFieldValue(ordinal); Assert.Equal(expectedValue, value); } } [Fact] public async Task ReadTupleColumn() { const string query = @"SELECT T.tval FROM ( SELECT tuple(cast(1 as Decimal(13, 4)), cast('one' as Nullable(String)), cast('1999-09-09 09:09:09' as Nullable(DateTime))) AS tval UNION ALL SELECT tuple(2, 'two', cast('2019-12-11 16:55:54' as DateTime('Asia/Yekaterinburg'))) UNION ALL SELECT tuple(3, null, cast('2007-01-11 05:32:48' as DateTime))) T ORDER BY T.tval.3"; await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(query); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetFieldValue>(0); DateTime dt; switch (count) { case 0: Assert.Equal(1, value.Item1); Assert.Equal("one", value.Item2); dt = new DateTime(1999, 9, 9, 9, 9, 9); Assert.Equal(dt, value.Item3); break; case 1: Assert.Equal(3, value.Item1); Assert.Null(value.Item2); dt = new DateTime(2007, 1, 11, 5, 32, 48); Assert.Equal(dt, value.Item3); break; case 2: Assert.Equal(2, value.Item1); Assert.Equal("two", value.Item2); var tz = TimeZoneHelper.GetTimeZoneInfo("Asia/Yekaterinburg"); dt = TimeZoneInfo.ConvertTime(new DateTime(2019, 12, 11, 16, 55, 54), tz, connection.GetServerTimeZone()); Assert.Equal(dt, value.Item3); break; default: Assert.False(true, "Too many rows."); break; } ++count; } Assert.Equal(3, count); } [Fact] public async Task ReadValueTupleColumn() { const string query = @"SELECT T.tval FROM ( SELECT tuple(cast(1 as Decimal(13, 4)), cast('one' as Nullable(String)), cast('1999-09-09 09:09:09' as Nullable(DateTime))) AS tval UNION ALL SELECT tuple(2, 'two', cast('2019-12-11 16:55:54' as DateTime('Asia/Yekaterinburg'))) UNION ALL SELECT tuple(3, null, cast('2007-01-11 05:32:48' as DateTime))) T ORDER BY T.tval.3"; await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(query); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetFieldValue<(decimal number, string? str, DateTime? date)>(0); DateTime dt; switch (count) { case 0: Assert.Equal(1, value.number); Assert.Equal("one", value.str); dt = new DateTime(1999, 9, 9, 9, 9, 9); Assert.Equal(dt, value.date); break; case 1: Assert.Equal(3, value.number); Assert.Null(value.str); dt = new DateTime(2007, 1, 11, 5, 32, 48); Assert.Equal(dt, value.date); break; case 2: Assert.Equal(2, value.number); Assert.Equal("two", value.str); dt = new DateTime(2019, 12, 11, 16, 55, 54); dt = TimeZoneInfo.ConvertTime(dt, TimeZoneHelper.GetTimeZoneInfo("Asia/Yekaterinburg"), connection.GetServerTimeZone()); Assert.Equal(dt, value.date); break; default: Assert.False(true, "Too many rows."); break; } ++count; } Assert.Equal(3, count); } [Fact] public async Task ReadNamedTupleScalar() { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("SELECT CAST(('hello', 1, -1) AS Tuple(name String, id UInt32, `e \\`e\\` e` Int32))"); var result = await cmd.ExecuteScalarAsync<(string firstItem, uint secondItem, int thirdItem)>(); Assert.Equal(("hello", 1u, -1), result); } [Fact] public async Task SkipTupleColumn() { const string query = @"SELECT T.tval FROM ( SELECT tuple(cast(1 as Decimal(13, 4)), cast('one' as Nullable(String)), cast('1999-09-09 09:09:09' as Nullable(DateTime))) AS tval UNION ALL SELECT tuple(2, 'two', cast('2019-12-11 16:55:54' as DateTime('Asia/Yekaterinburg'))) UNION ALL SELECT tuple(3, null, cast('2007-01-11 05:32:48' as DateTime))) T ORDER BY T.tval.3"; await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(query); await cmd.ExecuteNonQueryAsync(); } [Fact] public async Task ReadIpV4Column() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ip4_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE ip4_test(val IPv4, strVal String) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO ip4_test(val, strVal) VALUES ('116.253.40.133','116.253.40.133')('10.0.151.56','10.0.151.56')('192.0.121.234','192.0.121.234')"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT val, strVal FROM ip4_test"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var ipAddr = reader.GetFieldValue(0); var ipAddrStr = reader.GetFieldValue(1); var expectedIpAddr = IPAddress.Parse(ipAddrStr); Assert.Equal(expectedIpAddr, ipAddr); ++count; } } Assert.Equal(3, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ip4_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task ReadIpV6Column() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ip6_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE ip6_test(val IPv6, strVal String) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO ip6_test(val, strVal) VALUES ('2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d','2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d')('2a02:aa08:e000:3100::2','2a02:aa08:e000:3100::2')('::ffff:192.0.121.234','::ffff:192.0.121.234')"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT val, strVal FROM ip6_test"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var ipAddr = reader.GetFieldValue(0); var ipAddrStr = reader.GetFieldValue(1); var expectedIpAddr = IPAddress.Parse(ipAddrStr); Assert.Equal(expectedIpAddr, ipAddr); ++count; } } Assert.Equal(3, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS ip6_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task ReadLowCardinalityColumn() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE low_cardinality_test(id Int32, str LowCardinality(String)) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT * FROM low_cardinality_test"; await using (var reader = cmd.ExecuteReader()) { Assert.False(await reader.ReadAsync()); } cmd.CommandText = "INSERT INTO low_cardinality_test(id, str) VALUES (1,'foo')(2,'bar')(4,'bar')(6,'bar')(3,'foo')(7,'foo')(8,'bar')(5,'foobar')"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT id, str FROM low_cardinality_test"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1); var expected = id == 5 ? "foobar" : id % 2 == 1 ? "foo" : "bar"; Assert.Equal(expected, str); ++count; } } Assert.Equal(8, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task ReadNullableLowCardinalityColumn() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_null_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE low_cardinality_null_test(id Int32, str LowCardinality(Nullable(String))) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT * FROM low_cardinality_null_test"; await using (var reader = cmd.ExecuteReader()) { Assert.False(await reader.ReadAsync()); } cmd.CommandText = "INSERT INTO low_cardinality_null_test(id, str) SELECT number, number%50 == 0 ? NULL : toString(number%200) FROM system.numbers LIMIT 30000"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO low_cardinality_null_test(id, str) SELECT number, number%50 == 0 ? NULL : toString(number%200) FROM system.numbers WHERE number>=30000 LIMIT 30000"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO low_cardinality_null_test(id, str) SELECT number, number%50 == 0 ? NULL : toString(number%400) FROM system.numbers WHERE number>=60000 LIMIT 30000"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT id, str FROM low_cardinality_null_test"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1, null); if (id % 50 == 0) Assert.Null(str); else if (id < 60000) Assert.Equal((id % 200).ToString(), str); else Assert.Equal((id % 400).ToString(), str); ++count; } } Assert.Equal(90000, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_null_test"); await cmd.ExecuteNonQueryAsync(); } } [Fact] public async Task ReadNullableStringLowCardinalityColumnAsArray() { await WithTemporaryTable("low_cardinality_as_array", "id Int32, str LowCardinality(Nullable(String))", Test); async Task Test(ClickHouseConnection connection, string tableName) { var stringValues = new string?[] { null, string.Empty, "фываasdf", "abcdef", "ghijkl", "null", "пролджэzcvgb", "ячсмить" }; var byteValues = stringValues.Select(v => v == null ? null : Encoding.UTF8.GetBytes(v)).ToArray(); const int rowCount = 150; await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, str) VALUES", CancellationToken.None)) { writer.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.UTF8)); await writer.WriteTableAsync(new object[] { Enumerable.Range(0, rowCount), Enumerable.Range(0, rowCount).Select(i => stringValues[i % stringValues.Length]) }, rowCount, CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, str FROM {tableName} ORDER BY id"); int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn(1, new ClickHouseColumnSettings(Encoding.UTF8)); char[] charBuffer = new char[stringValues.Select(v => v?.Length ?? 0).Max() + 7]; byte[] byteBuffer = new byte[byteValues.Select(v => v?.Length ?? 0).Max() + 3]; while(await reader.ReadAsync()) { var id = reader.GetInt32(0); Assert.Equal(count, id); var expectedStr = stringValues[id % stringValues.Length]; var valueAsCharArray = reader.GetFieldValue(1, null); Assert.Equal(expectedStr, valueAsCharArray == null ? null : new string(valueAsCharArray)); var valueAsByteArray = reader.GetFieldValue(1, null); var expectedByteArray = byteValues[id % byteValues.Length]; Assert.Equal(expectedByteArray, valueAsByteArray); if (expectedStr == null) { Assert.True(reader.IsDBNull(1)); } else { var len = (int)reader.GetChars(1, 0, charBuffer, 0, charBuffer.Length); Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(0, len))); len = (int)reader.GetBytes(1, 0, byteBuffer, 0, byteBuffer.Length); Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Take(len)); len = 0; while (len < charBuffer.Length - 7) { var size = Math.Min(3, charBuffer.Length - len - 7); var currentLen = (int)reader.GetChars(1, len, charBuffer, len + 7, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(7, len))); len = 0; while (len < byteBuffer.Length - 3) { var size = Math.Min(3, byteBuffer.Length - len - 3); var currentLen = (int)reader.GetBytes(1, len, byteBuffer, len + 3, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Skip(3).Take(len)); } ++count; } Assert.Equal(rowCount, count); } } [Fact] public async Task ReadStringLowCardinalityColumnAsArray() { await WithTemporaryTable("low_cardinality_not_null_as_array", "id Int32, str LowCardinality(String)", Test); async Task Test(ClickHouseConnection connection, string tableName) { var stringValues = new string[] { string.Empty, "фываasdf", "abcdef", "ghijkl", "null", "пролджэzcvgb", "ячсмить" }; var byteValues = stringValues.Select(v => Encoding.UTF8.GetBytes(v)).ToArray(); const int rowCount = 150; await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, str) VALUES", CancellationToken.None)) { writer.ConfigureColumn("str", new ClickHouseColumnSettings(Encoding.UTF8)); await writer.WriteTableAsync(new object[] { Enumerable.Range(0, rowCount), Enumerable.Range(0, rowCount).Select(i => stringValues[i % stringValues.Length]) }, rowCount, CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, str FROM {tableName} ORDER BY id"); int count = 0; await using var reader = await cmd.ExecuteReaderAsync(); reader.ConfigureColumn(1, new ClickHouseColumnSettings(Encoding.UTF8)); char[] charBuffer = new char[stringValues.Select(v => v.Length).Max() + 7]; byte[] byteBuffer = new byte[byteValues.Select(v => v.Length).Max() + 3]; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); Assert.Equal(count, id); var expectedStr = stringValues[id % stringValues.Length]; var valueAsCharArray = reader.GetFieldValue(1); Assert.Equal(expectedStr, new string(valueAsCharArray)); var valueAsByteArray = reader.GetFieldValue(1); var expectedByteArray = byteValues[id % byteValues.Length]; Assert.Equal(expectedByteArray, valueAsByteArray); var len = (int)reader.GetChars(1, 0, charBuffer, 0, charBuffer.Length); Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(0, len))); len = (int)reader.GetBytes(1, 0, byteBuffer, 0, byteBuffer.Length); Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Take(len)); len = 0; while (len < charBuffer.Length - 7) { var size = Math.Min(3, charBuffer.Length - len - 7); var currentLen = (int)reader.GetChars(1, len, charBuffer, len + 7, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedStr.Length, len); Assert.Equal(expectedStr, new string(((ReadOnlySpan)charBuffer).Slice(7, len))); len = 0; while (len < byteBuffer.Length - 3) { var size = Math.Min(3, byteBuffer.Length - len - 3); var currentLen = (int)reader.GetBytes(1, len, byteBuffer, len + 3, size); len += currentLen; if (currentLen < size) break; } Assert.Equal(expectedByteArray!.Length, len); Assert.Equal(expectedByteArray, byteBuffer.Skip(3).Take(len)); ++count; } Assert.Equal(rowCount, count); } } [Fact] public async Task SkipLowCardinalityColumn() { try { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_skip_test"); await cmd.ExecuteNonQueryAsync(); cmd = connection.CreateCommand("CREATE TABLE low_cardinality_skip_test(id Int32, str LowCardinality(Nullable(String))) ENGINE=Memory"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO low_cardinality_skip_test(id, str) SELECT number, number%50 == 0 ? NULL : toString(number%200) FROM system.numbers LIMIT 10000"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "INSERT INTO low_cardinality_skip_test(id, str) SELECT number, number%50 == 0 ? NULL : toString(number%200) FROM system.numbers WHERE number>=10000 LIMIT 10000"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SELECT id, str FROM low_cardinality_skip_test"; int count = 0; await using (var reader = cmd.ExecuteReader()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var str = reader.GetString(1, null); if (id % 50 == 0) Assert.Null(str); else Assert.Equal((id % 200).ToString(), str); if (++count == 100) break; } } Assert.Equal(100, count); cmd.CommandText = "SELECT count(*) FROM low_cardinality_skip_test"; count = (int) await cmd.ExecuteScalarAsync(); Assert.Equal(20000, count); } finally { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand("DROP TABLE IF EXISTS low_cardinality_skip_test"); await cmd.ExecuteNonQueryAsync(); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadFixedStringParameterScalar(ClickHouseParameterMode parameterMode) { var values = new[] {string.Empty, "0", "12345678", "abcdefg", "1234", "abcd", "абвг"}; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") {DbType = DbType.StringFixedLength, Size = 8}; cmd.Parameters.Add(param); foreach (var testValue in values) { param.Value = testValue; var value = await cmd.ExecuteScalarAsync(); var len = value.Length - value.Reverse().TakeWhile(b => b == 0).Count(); var strValue = Encoding.UTF8.GetString(value, 0, len); Assert.Equal(testValue, strValue); } param.Value = "123456789"; var exception = await Assert.ThrowsAnyAsync(() => cmd.ExecuteScalarAsync()); Assert.Equal(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, exception.ErrorCode); param.Value = "абвг0"; exception = await Assert.ThrowsAnyAsync(() => cmd.ExecuteScalarAsync()); Assert.Equal(ClickHouseErrorCodes.InvalidQueryParameterConfiguration, exception.ErrorCode); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadRandomFixedStringParameterScalar(ClickHouseParameterMode parameterMode) { Memory parameterValue = new byte[11]; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") { DbType = DbType.StringFixedLength, Size = parameterValue.Length, Value = parameterValue }; cmd.Parameters.Add(param); var random = new Random(); for (int i = 0; i < 512;) { if (i <= 256) { for (int j = 0; j < parameterValue.Length; j++) parameterValue.Span[j] = (byte)(i++ % 256); } else { for (int j = 0; j < parameterValue.Length; i++, j++) parameterValue.Span[j] = (byte)random.Next(257); } var value = await cmd.ExecuteScalarAsync(); Assert.Equal(parameterValue.ToArray(), value); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadGuidParameterScalar(ClickHouseParameterMode parameterMode) { var parameterValue = Guid.Parse("7FCFFE2D-E9A6-49E0-B8ED-9617603F5584"); await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") { DbType = DbType.Guid }; cmd.Parameters.Add(param); param.Value = parameterValue; var result = await cmd.ExecuteScalarAsync(); Assert.Equal(parameterValue, result); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDecimalParameterScalar(ClickHouseParameterMode parameterMode) { // The default ClickHouse type for decimal is Decimal128(9) const decimal minValueByDefault = 1m / 1_000_000_000; const decimal binarySparseValue = 281479271677952m * 4294967296m; var testData = new[] { decimal.Zero, decimal.One, decimal.MinusOne, decimal.MinValue, decimal.MaxValue, decimal.MinValue / 100, decimal.MaxValue / 100, decimal.One / 100, decimal.MinusOne / 100, minValueByDefault, -minValueByDefault, minValueByDefault / 10, -minValueByDefault / 10, binarySparseValue, -binarySparseValue }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") {DbType = DbType.Decimal}; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); var resultDecimal = Assert.IsType(result); if (Math.Abs(testValue) >= minValueByDefault) Assert.Equal(testValue, resultDecimal); else Assert.Equal(0, resultDecimal); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadCurrencyParameterScalar(ClickHouseParameterMode parameterMode) { const decimal maxCurrencyValue = 922_337_203_685_477.5807m, minCurrencyValue = -922_337_203_685_477.5808m, binarySparseValue = 7_205_759_833_289.5232m, currencyEpsilon = 0.0001m; var testData = new[] { decimal.Zero, decimal.One, decimal.MinusOne, minCurrencyValue, maxCurrencyValue, decimal.One / 100, decimal.MinusOne / 100, binarySparseValue, -binarySparseValue, currencyEpsilon, -currencyEpsilon, currencyEpsilon / 10, -currencyEpsilon / 10 }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") { DbType = DbType.Currency }; cmd.Parameters.Add(param); param.Value = minCurrencyValue - currencyEpsilon; var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = maxCurrencyValue + currencyEpsilon; handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); var resultDecimal = Assert.IsType(result); if (Math.Abs(testValue) >= currencyEpsilon) Assert.Equal(testValue, resultDecimal); else Assert.Equal(0, resultDecimal); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadVarNumericParameter(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param} AS p, toString(p)"); var param = new ClickHouseParameter("param") {DbType = DbType.VarNumeric, Precision = 7, Scale = 3}; cmd.Parameters.Add(param); { const decimal epsilon = 0.001m, formalMax = 9_999.999m, actualMax = 2_147_483.647m, actualMin = -2_147_483.648m; var values = new[] {decimal.Zero, decimal.One, decimal.MinusOne, epsilon, -epsilon, formalMax, -formalMax, actualMax, actualMin, epsilon / 10, -epsilon / 10}; foreach (var testValue in values) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var resultDecimal = reader.GetDecimal(0); var resultStr = reader.GetString(1); Assert.False(await reader.ReadAsync()); if (resultStr.StartsWith("--")) { Assert.True(testValue < -formalMax); resultStr = resultStr.Substring(1); } var parsedValue = decimal.Parse(resultStr, CultureInfo.InvariantCulture); if (Math.Abs(testValue) >= epsilon) { Assert.Equal(testValue, resultDecimal); Assert.Equal(testValue, parsedValue); } else { Assert.Equal(0, resultDecimal); Assert.Equal(0, parsedValue); } } param.Value = actualMax + epsilon; var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = actualMin - epsilon; handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } param.Precision = 18; param.Scale = 6; { const decimal epsilon = 0.000_001m, formalMax = 999_999_999_999.999_999m, actualMax = 9_223_372_036_854.775807m, actualMin = -9_223_372_036_854.775808m; var values = new[] { decimal.Zero, decimal.One, decimal.MinusOne, epsilon, -epsilon, formalMax, -formalMax, actualMax, actualMin, epsilon / 10, -epsilon / 10 }; foreach (var testValue in values) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var resultDecimal = reader.GetDecimal(0); var resultStr = reader.GetString(1); Assert.False(await reader.ReadAsync()); if (resultStr.StartsWith("--")) { Assert.True(testValue < -formalMax); resultStr = resultStr.Substring(1); } var parsedValue = decimal.Parse(resultStr, CultureInfo.InvariantCulture); if (Math.Abs(testValue) >= epsilon) { Assert.Equal(testValue, resultDecimal); Assert.Equal(testValue, parsedValue); } else { Assert.Equal(0, resultDecimal); Assert.Equal(0, parsedValue); } } param.Value = actualMax + epsilon; var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = actualMin - epsilon; handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } param.Precision = 35; param.Scale = 30; { const decimal formalMax = 99_999.999_999_999_999_999_999_999_99m, actualMax = 170_141_183.460_469_231_731_687_303_71m, actualMin = -actualMax, epsilon= 0.000_000_000_000_000_000_01m; var values = new[] {decimal.Zero, decimal.One, decimal.MinusOne, epsilon, -epsilon, formalMax, -formalMax, actualMax, actualMin}; foreach (var testValue in values) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var resultDecimal = reader.GetDecimal(0); var resultStr = reader.GetString(1); Assert.False(await reader.ReadAsync()); if (resultStr.StartsWith("--")) { Assert.True(testValue < -formalMax); resultStr = resultStr.Substring(1); } var parsedValue = decimal.Parse(resultStr, CultureInfo.InvariantCulture); Assert.Equal(testValue, resultDecimal); Assert.Equal(testValue, parsedValue); } param.Value = actualMax + epsilon; var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = actualMin - epsilon; handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ClickHouseDecimalTypeNames(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT toTypeName({param})"); var param = new ClickHouseParameter("param") {DbType = DbType.Decimal, Value = 0m}; cmd.Parameters.Add(param); var typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(38, 9), typeName); param.DbType = DbType.Currency; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(18, 4), typeName); param.DbType = DbType.VarNumeric; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(38, 9), typeName); param.Scale = 2; param.Precision = 3; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(3, 2), typeName); param.Scale = 14; param.Precision = 14; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(14, 14), typeName); param.Scale = 0; param.Precision = 1; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(1, 0), typeName); param.Scale = 32; param.Precision = 33; typeName = await cmd.ExecuteScalarAsync(); Assert.Matches(GetRegex(33, 32), typeName); static string GetRegex(int precision, int scale) { // ClickHouse server may wrap the parameter's type into Nullable return string.Format(CultureInfo.InvariantCulture, @"^(Nullable\()?Decimal\({0},\s{1}\)\)?$", precision, scale); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateTimeParameterScalar(ClickHouseParameterMode parameterMode) { var now = DateTime.Now; now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Kind); var testData = new[] {now, default, new DateTime(1980, 12, 15, 3, 8, 58), new DateTime(2015, 1, 1, 18, 33, 55)}; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param"); cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); Assert.Equal(testValue, result); } param.Value = DateTime.UnixEpoch.AddMonths(-1); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = DateTime.UnixEpoch.AddSeconds(uint.MaxValue).AddMonths(1); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateTimeOffsetParameterScalar(ClickHouseParameterMode parameterMode) { var now = DateTime.Now; now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Kind); var testData = new[] { now, default, new DateTimeOffset(new DateTime(1980, 12, 15, 3, 8, 58), new TimeSpan(0, -5, 0, 0)), new DateTimeOffset(new DateTime(2015, 1, 1, 18, 33, 55), new TimeSpan(0, 3, 15, 0)), new DateTimeOffset(DateTime.UnixEpoch.AddSeconds(1)).ToOffset(new TimeSpan(0, -11, 0, 0)), new DateTimeOffset(DateTime.UnixEpoch.AddSeconds(uint.MaxValue)).ToOffset(new TimeSpan(0, 11, 0, 0)) }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param"); cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); Assert.Equal(0, (result - testValue).Ticks); } param.Value = DateTimeOffset.UnixEpoch; var unixEpochResult = await cmd.ExecuteScalarAsync(); Assert.Equal(default, unixEpochResult); param.Value = new DateTimeOffset(DateTime.UnixEpoch.AddSeconds(-1)).ToOffset(new TimeSpan(0, -11, 0, 0)); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = new DateTimeOffset(DateTime.UnixEpoch.AddSeconds((double) uint.MaxValue + 1)).ToOffset(new TimeSpan(0, 11, 0, 0)); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadFloatParameterScalar(ClickHouseParameterMode parameterMode) { var testData = new[] {float.MinValue, float.MaxValue, float.Epsilon * 2, -float.Epsilon * 2, 1, -1, (float) Math.PI, (float) Math.Exp(1)}; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT CAST({param}/2 AS Float32)"); var param = new ClickHouseParameter("param") { DbType = DbType.Single }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); var resultFloat = Assert.IsType(result); Assert.Equal(testValue / 2, resultFloat); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDoubleParameterScalar(ClickHouseParameterMode parameterMode) { var testData = new[] {Math.Exp(1), double.MinValue, double.MaxValue, double.Epsilon * 2, -double.Epsilon * 2, 1, -1, Math.PI, }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}/2"); var param = new ClickHouseParameter("param") { DbType = DbType.Double }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; var result = await cmd.ExecuteScalarAsync(); var resultDouble = Assert.IsType(result); Assert.Equal(testValue / 2, resultDouble); } } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadNothingParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param"); cmd.Parameters.Add(param); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(DBNull.Value, result); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadIpV4ParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") {Value = IPAddress.Parse("10.0.121.1")}; Assert.Equal(ClickHouseDbType.IpV4, param.ClickHouseDbType); cmd.Parameters.Add(param); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(param.Value, result); param.Value = "::ffff:192.0.2.1"; param.ClickHouseDbType = ClickHouseDbType.IpV4; result = await cmd.ExecuteScalarAsync(); Assert.Equal("192.0.2.1", result); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadIpV6ParameterScalar(ClickHouseParameterMode parameterMode) { await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param}"); var param = new ClickHouseParameter("param") { Value = IPAddress.Parse("2001:0db8:11a3:09d7:1f34:8a2e:07a0:765d") }; Assert.Equal(ClickHouseDbType.IpV6, param.ClickHouseDbType); cmd.Parameters.Add(param); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(param.Value, result); param.Value = "192.0.121.234"; param.ClickHouseDbType = ClickHouseDbType.IpV6; result = await cmd.ExecuteScalarAsync(); Assert.Equal("::ffff:192.0.121.234", result); param.Value = null; result = await cmd.ExecuteScalarAsync(); Assert.Equal(DBNull.Value, result); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadIntegerParameterScalar(ClickHouseParameterMode parameterMode) { object[] values = { sbyte.MinValue, sbyte.MaxValue, (sbyte) 0, (short) sbyte.MinValue, (short) sbyte.MaxValue, (int) sbyte.MinValue, (int) sbyte.MaxValue, (long) sbyte.MinValue, (long) sbyte.MaxValue, byte.MinValue, byte.MaxValue, (ushort) byte.MaxValue, (uint) byte.MaxValue, (ulong) byte.MaxValue, short.MinValue, short.MaxValue, (short) 0, (int) short.MinValue, (int) short.MaxValue, (long) short.MinValue, (long) short.MaxValue, ushort.MinValue, ushort.MaxValue, (uint) ushort.MaxValue, (ulong) ushort.MaxValue, int.MinValue, int.MaxValue, (int) 0, (long) int.MinValue, (long) int.MaxValue, uint.MinValue, uint.MaxValue, (ulong) uint.MaxValue, long.MinValue, long.MaxValue, (long) 0, ulong.MinValue, ulong.MaxValue }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {integerParam}"); var param = new ClickHouseParameter("integerParam"); cmd.Parameters.Add(param); foreach (var value in values) { param.Value = value; var result = await cmd.ExecuteScalarAsync(); Assert.IsType(value.GetType(), result); Assert.Equal(value, result); } } [Fact] public async Task ReadValueWithOverridenType() { await WithTemporaryTable("col_settings_type", "id UInt16, ip Nullable(IPv4), enum Nullable(Enum16('min'=-512, 'avg'=0, 'max'=512)), num Nullable(Int32)", RunTest); static async Task RunTest(ClickHouseConnection connection, string tableName) { var cmd = connection.CreateCommand($"INSERT INTO {tableName}(id, ip, enum, num) VALUES" + "(124, null, 'min', 1234)" + "(125, '10.0.0.1', 'avg', null)" + "(126, null, null, null)" + "(127, '127.0.0.1', 'max', -8990)" + "(128, '4.8.15.16', null, 12789)"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = $"SELECT * FROM {tableName} ORDER BY id"; await using var reader = await cmd.ExecuteReaderAsync(); var idIdx = reader.GetOrdinal("id"); var numIdx = reader.GetOrdinal("num"); var enumIdx = reader.GetOrdinal("enum"); var ipIdx = reader.GetOrdinal("ip"); Assert.Equal(typeof(ushort), reader.GetFieldType(idIdx)); Assert.Equal(typeof(int), reader.GetFieldType(numIdx)); Assert.Equal(typeof(string), reader.GetFieldType(enumIdx)); Assert.Equal(typeof(IPAddress), reader.GetFieldType(ipIdx)); reader.ConfigureColumn("id", new ClickHouseColumnSettings(typeof(int))); reader.ConfigureColumn("num", new ClickHouseColumnSettings(typeof(long))); reader.ConfigureColumn("enum", new ClickHouseColumnSettings(typeof(short?))); reader.ConfigureColumn("ip", new ClickHouseColumnSettings(typeof(string))); Assert.Equal(typeof(int), reader.GetFieldType(idIdx)); Assert.Equal(typeof(long), reader.GetFieldType(numIdx)); Assert.Equal(typeof(short), reader.GetFieldType(enumIdx)); Assert.Equal(typeof(string), reader.GetFieldType(ipIdx)); var expectedData = new object[][] { new object[] {124, DBNull.Value, (short)-512, 1234L}, new object[] {125, "10.0.0.1", (short)0, DBNull.Value}, new object[] {126, DBNull.Value, DBNull.Value, DBNull.Value}, new object[] {127, "127.0.0.1", (short)512, -8990L}, new object[] {128, "4.8.15.16", DBNull.Value, 12789L} }; int count = 0; while(await reader.ReadAsync()) { Assert.Equal(expectedData[count][0], reader.GetValue(idIdx)); Assert.Equal(expectedData[count][1], reader.GetValue(ipIdx)); Assert.Equal(expectedData[count][2], reader.GetValue(enumIdx)); Assert.Equal(expectedData[count][3], reader.GetValue(numIdx)); ++count; } Assert.Equal(expectedData.Length, count); } } [Fact] public async Task ReadGuidColumn() { var guids = new List { Guid.Parse("74D47928-2423-4FE2-AD45-82E296BF6058"), Guid.Parse("2879D474-2324-E24F-AD45-82E296BF6058"), Guid.Empty }; guids.AddRange(Enumerable.Range(1, 100 - guids.Count).Select(_ => Guid.NewGuid())); await WithTemporaryTable("uuid", "id Int32, guid UUID, str String", RunTest); async Task RunTest(ClickHouseConnection connection, string tableName) { await using (var writer = await connection.CreateColumnWriterAsync($"INSERT INTO {tableName}(id, guid, str) VALUES", CancellationToken.None)) { await writer.WriteTableAsync(new object[] { Enumerable.Range(0, guids.Count), guids, guids.Select(v => v.ToString("D")) }, guids.Count, CancellationToken.None); } var cmd = connection.CreateCommand($"SELECT id, guid, CAST(guid AS String) strGuid, (guid = CAST(str AS UUID)) eq FROM {tableName} ORDER BY id"); await using var reader = await cmd.ExecuteReaderAsync(); int count = 0; while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var guid = reader.GetGuid(1); var str = reader.GetString(2); var eq = reader.GetBoolean(3); Assert.Equal(count, id); Assert.Equal(guids[id], guid); Assert.True(Guid.TryParse(str, out var strGuid)); Assert.Equal(guids[id], strGuid); Assert.True(eq); ++count; } Assert.Equal(guids.Count, count); } } [Fact] public async Task ReadMapScalar() { await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT CAST(([1, 2, 3, 0], ['Ready', 'Steady', 'Go', null]), 'Map(UInt8, Nullable(String))') AS map"); var result = await cmd.ExecuteScalarAsync(); Assert.Equal(new[] { new KeyValuePair(1, "Ready"), new KeyValuePair(2, "Steady"), new KeyValuePair(3, "Go"), new KeyValuePair(0, null) }, result); } [Fact] public async Task ReadMapColumn() { await WithTemporaryTable("map", "id Int32, map Map(String, Nullable(Int32))", Test); static async Task Test(ClickHouseConnection cn, string tableName) { var expectedDict = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["Ready"] = 1, ["Steady"] = 2, ["Go"] = 3, ["Unknown"] = null, ["Ok"] = 1, ["NotOk"] = -1, ["Oh"] = 199, ["hh"] = null, ["hO"] = 201, ["null"] = null, }; var expectedLists = new string[][] { new string[]{ "Ready", "Steady", "Go", "Unknown" }, new string[]{ "Ok", "NotOk" }, new string[]{ "Oh", "hh", "hO" }, new string[0], new string[]{ "null" }, }; var cmd = cn.CreateCommand(@$"INSERT INTO {tableName}(id, map) SELECT 1, CAST((['Ready', 'Steady', 'Go', 'Unknown'], [1, 2, 3, null]), 'Map(String, Nullable(Int32))') AS map UNION ALL SELECT 2, CAST((['Ok', 'NotOk'], [1, -1]), 'Map(String, Nullable(Int32))') UNION ALL SELECT 3, CAST((['Oh', 'hh', 'hO'], [199, null, 201]), 'Map(String, Nullable(Int32))') UNION ALL SELECT 4, CAST(([], []), 'Map(String, Nullable(Int32))') UNION ALL SELECT 5, CAST((['null'], [null]), 'Map(String, Nullable(Int32))')"); await cmd.ExecuteNonQueryAsync(); cmd.CommandText = $"SELECT map FROM {tableName} ORDER BY id"; await using var reader = await cmd.ExecuteReaderAsync(); var columnType = reader.GetFieldTypeInfo(0); Assert.Equal("Map", columnType.TypeName); Assert.Equal("Map(String, Nullable(Int32))", columnType.ComplexTypeName); Assert.Equal(2, columnType.GenericArgumentsCount); Assert.Equal(2, columnType.TypeArgumentsCount); var keyArg = columnType.GetGenericArgument(0); var typeArg1 = columnType.GetTypeArgument(0); Assert.Same(keyArg, typeArg1); Assert.Equal("String", keyArg.ComplexTypeName); var valueArg = columnType.GetGenericArgument(1); var typeArg2 = columnType.GetTypeArgument(1); Assert.Same(valueArg, typeArg2); Assert.Equal("Nullable(Int32)", valueArg.ComplexTypeName); Assert.Equal(typeof(KeyValuePair[]), columnType.GetFieldType()); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetValue(0); var pairs = Assert.IsType[]>(value); var expectedKeys = expectedLists[count]; Assert.Equal(expectedKeys.Length, pairs.Length); for (int i = 0; i < expectedKeys.Length; i++) { Assert.Equal(expectedKeys[i], pairs[i].Key); Assert.Equal(expectedDict[expectedKeys[i]], pairs[i].Value); } ++count; } Assert.Equal(expectedLists.Length, count); } } [Fact] public async Task ReadInt128Column() { var minValue = -(BigInteger.One << 127); var maxValue = (BigInteger.One << 127) - BigInteger.One; var maxStrLen = maxValue.ToString().Length; var sb = new StringBuilder(maxStrLen + 1).Append('-'); for (int i = 1; i <= maxStrLen; i++) sb.Append((char)('0' + (i % 10))); var strValues = new[] { minValue.ToString(), sb.ToString(), "-1", "0", "1", sb.ToString(1, maxStrLen), maxValue.ToString() }; await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT CAST(value AS Int128) AS v, v = bigIntValue AS testPassed FROM ptable ORDER BY id"); var tableProvider = new ClickHouseTableProvider("ptable", strValues.Length); tableProvider.Columns.AddColumn("id", Enumerable.Range(1, strValues.Length)); tableProvider.Columns.AddColumn("value", strValues); var bigIntColumn = tableProvider.Columns.AddColumn("bigIntValue", strValues.Select(v => BigInteger.Parse(v))); bigIntColumn.ClickHouseDbType = ClickHouseDbType.Int128; cmd.TableProviders.Add(tableProvider); await using var reader = await cmd.ExecuteReaderAsync(); var valueColumnType = reader.GetFieldTypeInfo(0); Assert.Equal("Int128", valueColumnType.ComplexTypeName); Assert.Equal(ClickHouseDbType.Int128, valueColumnType.GetDbType()); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetValue(0); var expectedValue = BigInteger.Parse(strValues[count]); Assert.Equal(expectedValue, value); var testPassed = reader.GetBoolean(1); Assert.True(testPassed); ++count; } Assert.Equal(strValues.Length, count); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadInt128ParameterScalar(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT {v}"); var parameter = new ClickHouseParameter("v") { ClickHouseDbType = ClickHouseDbType.Int128 }; cmd.Parameters.Add(parameter); var pairs = new (object value, BigInteger expected)[] { (0, 0), (-1, -1), (1, 1), (byte.MaxValue, byte.MaxValue), (sbyte.MinValue, sbyte.MinValue), (sbyte.MaxValue, sbyte.MaxValue), (short.MinValue, short.MinValue), (short.MaxValue, short.MaxValue), (ushort.MaxValue, ushort.MaxValue), (int.MinValue, int.MinValue), (int.MaxValue, int.MaxValue), (uint.MaxValue, uint.MaxValue), (long.MinValue, long.MinValue), (long.MaxValue, long.MaxValue), (ulong.MaxValue, ulong.MaxValue), (-(BigInteger.One << 127), -(BigInteger.One << 127)), ((BigInteger.One << 127) - 1, (BigInteger.One << 127) - 1), }; foreach(var pair in pairs) { parameter.Value = pair.value; var result = await cmd.ExecuteScalarAsync(); var bigIntResult = Assert.IsType(result); Assert.Equal(pair.expected, bigIntResult); } } [Fact] public async Task ReadUInt128Column() { var maxValue = (BigInteger.One << 128) - BigInteger.One; var maxStrLen = maxValue.ToString().Length; var sb = new StringBuilder(maxStrLen); for (int i = 1; i <= maxStrLen; i++) sb.Append((char)('0' + (i % 10))); var strValues = new[] { "0", "1", sb.ToString(), maxValue.ToString() }; await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT CAST(value AS UInt128) AS v, v = bigIntValue AS testPassed FROM ptable ORDER BY id"); var tableProvider = new ClickHouseTableProvider("ptable", strValues.Length); tableProvider.Columns.AddColumn("id", Enumerable.Range(1, strValues.Length)); tableProvider.Columns.AddColumn("value", strValues); var bigIntColumn = tableProvider.Columns.AddColumn("bigIntValue", strValues.Select(v => BigInteger.Parse(v))); bigIntColumn.ClickHouseDbType = ClickHouseDbType.UInt128; cmd.TableProviders.Add(tableProvider); await using var reader = await cmd.ExecuteReaderAsync(); var valueColumnType = reader.GetFieldTypeInfo(0); Assert.Equal("UInt128", valueColumnType.ComplexTypeName); Assert.Equal(ClickHouseDbType.UInt128, valueColumnType.GetDbType()); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetValue(0); var expectedValue = BigInteger.Parse(strValues[count]); Assert.Equal(expectedValue, value); var testPassed = reader.GetBoolean(1); Assert.True(testPassed); ++count; } Assert.Equal(strValues.Length, count); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadUInt128ParameterScalar(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT {v}"); var parameter = new ClickHouseParameter("v") { ClickHouseDbType = ClickHouseDbType.UInt128 }; cmd.Parameters.Add(parameter); var pairs = new (object value, BigInteger expected)[] { (0u, 0u), (1u, 1u), (byte.MaxValue, byte.MaxValue), (ushort.MaxValue, ushort.MaxValue), (uint.MaxValue, uint.MaxValue), (ulong.MaxValue, ulong.MaxValue), ((BigInteger.One << 128) - 1, (BigInteger.One << 128) - 1), }; foreach (var pair in pairs) { parameter.Value = pair.value; var result = await cmd.ExecuteScalarAsync(); var bigIntResult = Assert.IsType(result); Assert.Equal(pair.expected, bigIntResult); } } [Fact] public async Task ReadInt256Column() { var minValue = -(BigInteger.One << 255); var maxValue = (BigInteger.One << 255) - BigInteger.One; var maxStrLen = maxValue.ToString().Length; var sb = new StringBuilder(maxStrLen + 1).Append('-'); for (int i = 1; i <= maxStrLen; i++) sb.Append((char)('0' + (i % 10))); var strValues = new[] { minValue.ToString(), sb.ToString(), "-1", "0", "1", sb.ToString(1, maxStrLen), maxValue.ToString() }; await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT CAST(value AS Int256) AS v, v = bigIntValue AS testPassed FROM ptable ORDER BY id"); var tableProvider = new ClickHouseTableProvider("ptable", strValues.Length); tableProvider.Columns.AddColumn("id", Enumerable.Range(1, strValues.Length)); tableProvider.Columns.AddColumn("value", strValues); tableProvider.Columns.AddColumn("bigIntValue", strValues.Select(v => BigInteger.Parse(v))); cmd.TableProviders.Add(tableProvider); await using var reader = await cmd.ExecuteReaderAsync(); var valueColumnType = reader.GetFieldTypeInfo(0); Assert.Equal("Int256", valueColumnType.ComplexTypeName); Assert.Equal(ClickHouseDbType.Int256, valueColumnType.GetDbType()); int count = 0; while(await reader.ReadAsync()) { var value = reader.GetValue(0); var expectedValue = BigInteger.Parse(strValues[count]); Assert.Equal(expectedValue, value); var testPassed = reader.GetBoolean(1); Assert.True(testPassed); ++count; } Assert.Equal(strValues.Length, count); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadInt256ParameterScalar(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT {v}"); var parameter = new ClickHouseParameter("v") { ClickHouseDbType = ClickHouseDbType.Int256 }; cmd.Parameters.Add(parameter); var pairs = new (object value, BigInteger expected)[] { (0, 0), (-1, -1), (1, 1), (byte.MaxValue, byte.MaxValue), (sbyte.MinValue, sbyte.MinValue), (sbyte.MaxValue, sbyte.MaxValue), (short.MinValue, short.MinValue), (short.MaxValue, short.MaxValue), (ushort.MaxValue, ushort.MaxValue), (int.MinValue, int.MinValue), (int.MaxValue, int.MaxValue), (uint.MaxValue, uint.MaxValue), (long.MinValue, long.MinValue), (long.MaxValue, long.MaxValue), (ulong.MaxValue, ulong.MaxValue), (-(BigInteger.One << 255), -(BigInteger.One << 255)), ((BigInteger.One << 255) - 1, (BigInteger.One << 255) - 1), }; foreach (var pair in pairs) { if (pair.value is BigInteger) parameter.ResetDbType(); parameter.Value = pair.value; var result = await cmd.ExecuteScalarAsync(); var bigIntResult = Assert.IsType(result); Assert.Equal(pair.expected, bigIntResult); } } [Fact] public async Task ReadUInt256Column() { var maxValue = (BigInteger.One << 256) - BigInteger.One; var maxStrLen = maxValue.ToString().Length; var sb = new StringBuilder(maxStrLen); sb.Append("11"); for (int i = 3; i <= maxStrLen; i++) sb.Append((char)('0' + (i % 10))); var strValues = new[] { "0", "1", sb.ToString(), maxValue.ToString() }; await using var cn = await OpenConnectionAsync(); var cmd = cn.CreateCommand("SELECT CAST(value AS UInt256) AS v, v = bigIntValue AS testPassed FROM ptable ORDER BY id"); var tableProvider = new ClickHouseTableProvider("ptable", strValues.Length); tableProvider.Columns.AddColumn("id", Enumerable.Range(1, strValues.Length)); tableProvider.Columns.AddColumn("value", strValues); var bigIntColumn = tableProvider.Columns.AddColumn("bigIntValue", strValues.Select(v => BigInteger.Parse(v))); bigIntColumn.ClickHouseDbType = ClickHouseDbType.UInt256; cmd.TableProviders.Add(tableProvider); await using var reader = await cmd.ExecuteReaderAsync(); var valueColumnType = reader.GetFieldTypeInfo(0); Assert.Equal("UInt256", valueColumnType.ComplexTypeName); Assert.Equal(ClickHouseDbType.UInt256, valueColumnType.GetDbType()); int count = 0; while (await reader.ReadAsync()) { var value = reader.GetValue(0); var expectedValue = BigInteger.Parse(strValues[count]); Assert.Equal(expectedValue, value); var testPassed = reader.GetBoolean(1); Assert.True(testPassed); ++count; } Assert.Equal(strValues.Length, count); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadUInt256ParameterScalar(ClickHouseParameterMode parameterMode) { await using var cn = await OpenConnectionAsync(parameterMode); var cmd = cn.CreateCommand("SELECT {v}"); var parameter = new ClickHouseParameter("v") { ClickHouseDbType = ClickHouseDbType.UInt256 }; cmd.Parameters.Add(parameter); var pairs = new (object value, BigInteger expected)[] { (0u, 0u), (1u, 1u), (byte.MaxValue, byte.MaxValue), (ushort.MaxValue, ushort.MaxValue), (uint.MaxValue, uint.MaxValue), (ulong.MaxValue, ulong.MaxValue), ((BigInteger.One << 256) - 1, (BigInteger.One << 256) - 1), }; foreach (var pair in pairs) { parameter.Value = pair.value; var result = await cmd.ExecuteScalarAsync(); var bigIntResult = Assert.IsType(result); Assert.Equal(pair.expected, bigIntResult); } } [Fact] public async Task ReadArrayLowCardinality() { await WithTemporaryTable("arrlc", "id Int32, arr Array(LowCardinality(String))", Test); static async Task Test(ClickHouseConnection cn, string tableName) { var cmd = cn.CreateCommand($"SELECT * FROM {tableName}"); await using (var reader = await cmd.ExecuteReaderAsync()) Assert.False(await reader.ReadAsync()); cmd.CommandText = $"INSERT INTO {tableName}(id, arr) VALUES (1, [])"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = $"SELECT arr FROM {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly)) { // Skipping all rows Assert.False(await reader.ReadAsync()); } await using (var reader = await cmd.ExecuteReaderAsync()) { Assert.True(await reader.ReadAsync()); Assert.Equal(Array.Empty(), reader.GetValue(0)); Assert.False(await reader.ReadAsync()); } cmd.CommandText = $"INSERT INTO {tableName}(id, arr) VALUES (2, ['abc', 'def'])(3, ['def', 'ghi'])(4, ['def'])(5, ['ghi', 'abc'])"; await cmd.ExecuteNonQueryAsync(); var expectedValues = new Dictionary { [1] = Array.Empty(), [2] = new[] { "abc", "def" }, [3] = new[] { "def", "ghi" }, [4] = new[] { "def" }, [5] = new[] { "ghi", "abc" } }; cmd.CommandText = $"SELECT id, arr FROM {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly)) { // Skipping all rows Assert.False(await reader.ReadAsync()); } await using (var reader = await cmd.ExecuteReaderAsync()) { while (await reader.ReadAsync()) { var id = reader.GetInt32(0); var value = reader.GetValue(1); var arr = Assert.IsType(value); Assert.True(expectedValues.Remove(id, out var expectedArr)); Assert.Equal(expectedArr, arr); } } Assert.Empty(expectedValues); } } [Theory] [InlineData("2021-11-09", 2021, 11, 09)] [InlineData("1970-1-1", 0, 0, 0)] // Default value [InlineData("1970-1-2", 1970, 1, 2)] [InlineData("2149-06-06", 2149, 6, 6)] public async Task ReadDateScalar(string str, int year, int month, int day) { DateTime expectedDateTime = default; #if NET6_0_OR_GREATER DateOnly expectedDate = default; #endif if (year != 0 || month != 0 || day != 0) { expectedDateTime = new DateTime(year, month, day); #if NET6_0_OR_GREATER expectedDate = new DateOnly(year, month, day); #endif } await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT cast('{str}' AS Date)"); var result = await cmd.ExecuteScalarAsync(); DateTime resultDateTime; #if NET6_0_OR_GREATER var resultDateOnly = Assert.IsType(result); Assert.Equal(expectedDate, resultDateOnly); #else resultDateTime = Assert.IsType(result); Assert.Equal(expectedDateTime, resultDateTime); #endif resultDateTime = await cmd.ExecuteScalarAsync(); Assert.Equal(expectedDateTime, resultDateTime); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateParameterScalar(ClickHouseParameterMode parameterMode) { var now = DateTime.Now; now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Kind); var testData = new[] { now, default, new DateTime(1980, 12, 15, 3, 8, 58), new DateTime(2015, 1, 1, 18, 33, 55) }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param} v, toString(v)"); var param = new ClickHouseParameter("param") { ClickHouseDbType = ClickHouseDbType.Date }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var result = reader.GetFieldValue(0); Assert.Equal(testValue.Date, result); if (testValue == default) continue; var resultStr = reader.GetString(1); Assert.Equal(testValue.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), resultStr); } param.Value = DateTime.UnixEpoch.AddMonths(-1); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = DateTime.UnixEpoch.AddDays(ushort.MaxValue).AddMonths(1); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } #if NET6_0_OR_GREATER [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDateParameterScalarNet6(ClickHouseParameterMode parameterMode) { var nowDateTime = DateTime.Now; var now = new DateOnly(nowDateTime.Year, nowDateTime.Month, nowDateTime.Day); var testData = new[] { now, default, new DateOnly(1980, 12, 15), new DateOnly(2015, 1, 1) }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param} v, toString(v)"); var param = new ClickHouseParameter("param") { ClickHouseDbType = ClickHouseDbType.Date }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; Assert.Equal(ClickHouseDbType.Date, param.ClickHouseDbType); await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var result = reader.GetValue(0); var resultDateOnly = Assert.IsType(result); Assert.Equal(testValue, resultDateOnly); if (testValue == default) continue; var resultStr = reader.GetString(1); Assert.Equal(testValue.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), resultStr); } param.Value = DateOnly.FromDateTime(DateTime.UnixEpoch.AddMonths(-1)); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = DateOnly.FromDateTime(DateTime.UnixEpoch.AddDays(ushort.MaxValue).AddMonths(1)); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } #endif [Theory] [InlineData("2021-11-09", 2021, 11, 09)] [InlineData("1925-01-01", 1925, 1, 1)] [InlineData("1925-1-02", 1925, 1, 2)] [InlineData("1970-1-1", 1970, 1, 1)] [InlineData("2283-11-11", 2283, 11, 11)] [InlineData("1900-1-1", 1900, 1, 1)] // Min value [InlineData("2299-12-31", 2299, 12, 31)] // Max value public async Task ReadDate32Scalar(string str, int year, int month, int day) { DateTime expectedDateTime = default; #if NET6_0_OR_GREATER DateOnly expectedDate = default; #endif if (year != 0 || month != 0 || day != 0) { expectedDateTime = new DateTime(year, month, day); #if NET6_0_OR_GREATER expectedDate = new DateOnly(year, month, day); #endif } await using var connection = await OpenConnectionAsync(); await using var cmd = connection.CreateCommand($"SELECT cast('{str}' AS Date32)"); var result = await cmd.ExecuteScalarAsync(); DateTime resultDateTime; #if NET6_0_OR_GREATER var resultDateOnly = Assert.IsType(result); Assert.Equal(expectedDate, resultDateOnly); #else resultDateTime = Assert.IsType(result); Assert.Equal(expectedDateTime, resultDateTime); #endif resultDateTime = await cmd.ExecuteScalarAsync(); Assert.Equal(expectedDateTime, resultDateTime); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDate32ParameterScalar(ClickHouseParameterMode parameterMode) { var now = DateTime.Now; var minValue = new DateTime(1900, 1, 1); var maxValue = new DateTime(2299, 12, 31, 23, 59, 59, 999); now = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Kind); var testData = new[] { now, default, new DateTime(1919, 6, 28, 18, 19, 20), new DateTime(1980, 12, 15, 3, 8, 58), new DateTime(2015, 1, 1, 18, 33, 55), minValue.AddDays(1), maxValue, DateTime.UnixEpoch }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param} v, toString(v)"); var param = new ClickHouseParameter("param") { ClickHouseDbType = ClickHouseDbType.Date32 }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var result = reader.GetFieldValue(0); var expected = testValue.Date; if (expected == default) expected = minValue; Assert.Equal(expected, result); if (testValue == default) continue; var resultStr = reader.GetString(1); Assert.Equal(testValue.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), resultStr); } param.Value = minValue.AddMonths(-1); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = maxValue.AddMonths(1); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } #if NET6_0_OR_GREATER [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadDate32ParameterScalarNet6(ClickHouseParameterMode parameterMode) { var nowDateTime = DateTime.Now; var now = new DateOnly(nowDateTime.Year, nowDateTime.Month, nowDateTime.Day); var minValue = new DateOnly(1900, 1, 1); var maxValue = new DateOnly(2299, 12, 31); var testData = new[] { now, default, new DateOnly(1912, 6, 28), new DateOnly(1980, 12, 15), new DateOnly(2015, 1, 1), minValue.AddDays(1), maxValue, DateOnly.FromDateTime(DateTime.UnixEpoch) }; await using var connection = await OpenConnectionAsync(parameterMode); await using var cmd = connection.CreateCommand("SELECT {param} AS v, toString(v)"); var param = new ClickHouseParameter("param") { ClickHouseDbType = ClickHouseDbType.Date32 }; cmd.Parameters.Add(param); foreach (var testValue in testData) { param.Value = testValue; await using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); var result = reader.GetValue(0); var resultDateOnly = Assert.IsType(result); var expected = testValue; if (expected == default) expected = minValue; Assert.Equal(expected, resultDateOnly); if (testValue == default) continue; var resultStr = reader.GetString(1); Assert.Equal(testValue.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), resultStr); } param.Value = minValue.AddMonths(-1); var handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); param.Value = maxValue.AddMonths(1); handledException = await Assert.ThrowsAsync(() => cmd.ExecuteScalarAsync()); Assert.IsType(handledException.InnerException); } #endif [Fact] public async Task ReadMultidimensionalArrayLowCardinality() { await WithTemporaryTable("arrlc", "id Int32, arr Array(Array(Array(LowCardinality(Nullable(String)))))", Test); static async Task Test(ClickHouseConnection cn, string tableName) { var cmd = cn.CreateCommand($"SELECT * FROM {tableName}"); await using (var reader = await cmd.ExecuteReaderAsync()) Assert.False(await reader.ReadAsync()); cmd.CommandText = $"INSERT INTO {tableName}(id, arr) VALUES (1, [[[]]])"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = $"SELECT arr FROM {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly)) { // Skipping all rows Assert.False(await reader.ReadAsync()); } await using (var reader = await cmd.ExecuteReaderAsync()) { Assert.True(await reader.ReadAsync()); Assert.Equal(new[] { new[] { Array.Empty() } }, reader.GetValue(0)); Assert.False(await reader.ReadAsync()); } cmd.CommandText = $"INSERT INTO {tableName}(id, arr) VALUES " + "(2, [[['foo', 'bar'], [''], ['foo', 'bar', null, 'baz']]])" + "(3, [[['foo', 'bar'], ['bar']], []])" + "(4, [[['foo']], [[]], [['baz', null], ['bar', 'foo']]])" + "(5, [[[NULL]], [[], ['baz']]])" + "(6, [])" + "(7, [[]])"; await cmd.ExecuteNonQueryAsync(); var expectedValues = new Dictionary { [1] = new[] { new[] { Array.Empty() } }, [2] = new[] { new[] { new[] { "foo", "bar" }, new[] { string.Empty }, new[] { "foo", "bar", null, "baz" } } }, [3] = new[] { new[] { new[] { "foo", "bar" }, new[] { "bar" } }, Array.Empty() }, [4] = new[] { new[] { new[] { "foo" } }, new[] { Array.Empty() }, new[] { new[] { "baz", null }, new[] { "bar", "foo" } } }, [5] = new[] { new[] { new[] { default(string) } }, new[] { Array.Empty(), new[] { "baz" } } }, [6] = Array.Empty(), [7] = new[] { Array.Empty() } }; cmd.CommandText = $"SELECT id, arr FROM {tableName}"; await using (var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly)) { // Skipping all rows Assert.False(await reader.ReadAsync()); } await using (var reader = await cmd.ExecuteReaderAsync()) { while(await reader.ReadAsync()) { var id = reader.GetInt32(0); var value = reader.GetValue(1); var arr = Assert.IsType(value); Assert.True(expectedValues.Remove(id, out var expectedArr)); Assert.Equal(expectedArr, arr); } } Assert.Empty(expectedValues); } } [Theory] [InlineData("true::Bool", true)] [InlineData("false::Bool", false)] [InlineData("1::Bool", true)] [InlineData("0::Bool", false)] [InlineData("NULL::Nullable(Bool)", null)] public async Task ReadBoolScalar(string value, bool? expectedValue) { await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(); cmd.CommandText = $"SELECT {value}"; var result = await cmd.ExecuteScalarAsync(); Assert.Equal((object?)expectedValue ?? DBNull.Value, result); } [Theory] [MemberData(nameof(ParameterModes))] public async Task ReadBoolParameter(ClickHouseParameterMode parameterMode) { var testData = new[] { false, true, (object?)null, DBNull.Value }; await using var connection = await OpenConnectionAsync(parameterMode); var sb = new StringBuilder("SELECT "); var cmd = connection.CreateCommand(); for (int i = 0; i < testData.Length * 2; i++) { var value = testData[i % testData.Length]; ClickHouseParameter p; if (value is bool) p = cmd.Parameters.AddWithValue($"p{i + 1}", value); else p = cmd.Parameters.AddWithValue($"p{i + 1}", value, ClickHouseDbType.Boolean); if (i > 0) sb.Append(", "); if (i > testData.Length) p.ParameterMode = ClickHouseParameterMode.Interpolate; sb.Append($"{{p{i + 1}}} AS p{i + 1}"); } cmd.CommandText = sb.ToString(); await using var reader = await cmd.ExecuteReaderAsync(); Assert.Equal(testData.Length*2, reader.FieldCount); Assert.True(await reader.ReadAsync()); for (int i = 0; i < testData.Length * 2; i++) { Assert.Equal(ClickHouseDbType.Boolean, reader.GetFieldTypeInfo(i).GetDbType()); var value = testData[i % testData.Length]; if (value is bool boolVal) { var result = reader.GetValue(i); Assert.Equal(boolVal, result); } else { Assert.True(reader.IsDBNull(i)); } } Assert.False(await reader.ReadAsync()); } [Fact] public async Task ReadVariant() { const string query = "SELECT if(number = 1, NULL::Variant(Array(UInt64), UInt64), if(number % 2 != 0, number, range(number))) as variant FROM numbers(5)"; await using var connection = await OpenConnectionAsync(); var cmd = connection.CreateCommand(); // TODO: remove when this feature will be non-experimental cmd.CommandText = "SET allow_experimental_variant_type = 1"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = "SET use_variant_as_common_type = 1"; await cmd.ExecuteNonQueryAsync(); cmd.CommandText = query; await using var reader = await cmd.ExecuteReaderAsync(); ulong count = 0; while (await reader.ReadAsync()) { var value = reader.GetValue(0); if (count % 2 == 0) { Assert.False(reader.IsDBNull(0)); var arrValue = Assert.IsType(value); Assert.Equal(count, (ulong)arrValue.Length); for (ulong i = 0; i < count; i++) Assert.Equal(i, arrValue[i]); } else if (count == 1) { Assert.IsType(value); Assert.True(reader.IsDBNull(0)); } else { Assert.False(reader.IsDBNull(0)); var longVal = Assert.IsType(value); Assert.Equal(count, longVal); } ++count; } } [Fact] public async Task CreateInsertSelectAllKnownNullable() { const string ddl = @" CREATE TABLE clickhouse_test_nullable ( int8 Nullable(Int8), int16 Nullable(Int16), int32 Nullable(Int32), int64 Nullable(Int64), uint8 Nullable(UInt8), uint16 Nullable(UInt16), uint32 Nullable(UInt32), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fString Nullable(FixedString(2)), date Nullable(Date), datetime Nullable(DateTime), enum8 Nullable(Enum8 ('a' = 127, 'b' = 2)), enum16 Nullable(Enum16('c' = -32768, 'd' = 2, '' = 42)) ) Engine=Memory;"; const string dml = @" INSERT INTO clickhouse_test_nullable ( int8 ,int16 ,int32 ,int64 ,uint8 ,uint16 ,uint32 ,uint64 ,float32 ,float64 ,string ,fString ,date ,datetime ,enum8 ,enum16 ) SELECT 8 ,16 ,32 ,64 ,18 ,116 ,132 ,165 ,1.1 ,1.2 ,'RU' ,'UA' ,now() ,now() ,'a' ,'c'"; const string query = @" SELECT int8 ,int16 ,int32 ,int64 ,uint8 ,uint16 ,uint32 ,uint64 ,float32 ,float64 ,string ,fString ,date ,datetime ,enum8 ,enum16 FROM clickhouse_test_nullable"; try { await using var connection = await OpenConnectionAsync(); await using var cmdDrop = connection.CreateCommand("DROP TABLE IF EXISTS clickhouse_test_nullable "); var ddlResult = await cmdDrop.ExecuteNonQueryAsync(); Assert.Equal(0, ddlResult); await using var cmd = connection.CreateCommand(ddl); var result = await cmd.ExecuteNonQueryAsync(); Assert.Equal(0, result); await using var dmlcmd = connection.CreateCommand(dml); result = await dmlcmd.ExecuteNonQueryAsync(); if (!connection.ServerVersion.StartsWith("21.11.")) { // The server of the version 21.11 doesn't send profile events. Assert.Equal(1, result); } await using var queryCmd = connection.CreateCommand(query); var r = await queryCmd.ExecuteReaderAsync(); while (r.Read()) { Assert.Equal(typeof(sbyte), r.GetFieldType(0)); Assert.Equal((sbyte) 8, r.GetFieldValue(0)); //int8 Assert.Equal((sbyte) 8, r.GetValue(0)); Assert.Equal(typeof(short), r.GetFieldType(1)); Assert.Equal((short) 16, r.GetInt16(1)); Assert.Equal((short) 16, r.GetValue(1)); Assert.Equal(typeof(int), r.GetFieldType(2)); Assert.Equal((int) 32, r.GetInt32(2)); Assert.Equal((int) 32, r.GetValue(2)); Assert.Equal(typeof(long), r.GetFieldType(3)); Assert.Equal((long) 64, r.GetInt64(3)); Assert.Equal((long) 64, r.GetValue(3)); Assert.Equal(typeof(byte), r.GetFieldType(4)); Assert.Equal((byte) 18, r.GetFieldValue(4)); //uint8 Assert.Equal((byte) 18, r.GetValue(4)); Assert.Equal(typeof(ushort), r.GetFieldType(5)); Assert.Equal((ushort) 116, r.GetFieldValue(5)); Assert.Equal((ushort) 116, r.GetValue(5)); Assert.Equal(typeof(uint), r.GetFieldType(6)); Assert.Equal((uint) 132, r.GetFieldValue(6)); Assert.Equal((uint) 132, r.GetValue(6)); Assert.Equal(typeof(ulong), r.GetFieldType(7)); Assert.Equal((UInt64) 165, r.GetFieldValue(7)); Assert.Equal((UInt64) 165, r.GetValue(7)); Assert.Equal(typeof(float), r.GetFieldType(8)); Assert.Equal((float) 1.1, r.GetFloat(8)); Assert.Equal((float) 1.1, r.GetValue(8)); Assert.Equal(typeof(double), r.GetFieldType(9)); Assert.Equal((double) 1.2, r.GetDouble(9)); Assert.Equal((double) 1.2, r.GetValue(9)); Assert.Equal(typeof(string), r.GetFieldType(10)); Assert.Equal("RU", r.GetString(10)); Assert.Equal("RU", r.GetValue(10)); Assert.Equal(typeof(byte[]), r.GetFieldType(11)); var fixedStringBytes = r.GetFieldValue(11); var fixedStringBytesAsValue = r.GetValue(11) as byte[]; Assert.Equal(fixedStringBytes, fixedStringBytesAsValue); Assert.NotNull(fixedStringBytes as byte[]); Assert.Equal("UA", Encoding.Default.GetString(fixedStringBytes as byte[])); Assert.Equal(typeof(string), r.GetFieldType(14)); Assert.Equal("a", r.GetValue(14)); Assert.Equal(127, r.GetFieldValue(14)); Assert.Equal(typeof(string), r.GetFieldType(15)); Assert.Equal("c", r.GetValue(15)); Assert.Equal(-32768, r.GetInt16(15)); } } finally { await using var connection = await OpenConnectionAsync(); await using var cmdDrop = connection.CreateCommand("DROP TABLE IF EXISTS clickhouse_test_nullable "); await cmdDrop.ExecuteNonQueryAsync(); } } } } ================================================ FILE: src/Octonica.ClickHouseClient.Tests/xunit.runner.json ================================================ { "parallelizeAssembly": false, "parallelizeTestCollections": false } ================================================ FILE: src/Octonica.ClickHouseClient.sln ================================================  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.29319.158 MinimumVisualStudioVersion = 15.0.26124.0 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octonica.ClickHouseClient", "Octonica.ClickHouseClient\Octonica.ClickHouseClient.csproj", "{7D934752-DF65-47D4-B183-9D1D6A8E4BDA}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octonica.ClickHouseClient.Tests", "Octonica.ClickHouseClient.Tests\Octonica.ClickHouseClient.Tests.csproj", "{AEE192D3-405E-4804-9883-10A4F9FC902E}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Octonica.ClickHouseClient.Benchmarks", "Octonica.ClickHouseClient.Benchmarks\Octonica.ClickHouseClient.Benchmarks.csproj", "{71FA7684-9E05-4F20-A45C-8509CCA4E2D5}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {7D934752-DF65-47D4-B183-9D1D6A8E4BDA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7D934752-DF65-47D4-B183-9D1D6A8E4BDA}.Debug|Any CPU.Build.0 = Debug|Any CPU {7D934752-DF65-47D4-B183-9D1D6A8E4BDA}.Release|Any CPU.ActiveCfg = Release|Any CPU {7D934752-DF65-47D4-B183-9D1D6A8E4BDA}.Release|Any CPU.Build.0 = Release|Any CPU {AEE192D3-405E-4804-9883-10A4F9FC902E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AEE192D3-405E-4804-9883-10A4F9FC902E}.Debug|Any CPU.Build.0 = Debug|Any CPU {AEE192D3-405E-4804-9883-10A4F9FC902E}.Release|Any CPU.ActiveCfg = Release|Any CPU {AEE192D3-405E-4804-9883-10A4F9FC902E}.Release|Any CPU.Build.0 = Release|Any CPU {71FA7684-9E05-4F20-A45C-8509CCA4E2D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {71FA7684-9E05-4F20-A45C-8509CCA4E2D5}.Debug|Any CPU.Build.0 = Debug|Any CPU {71FA7684-9E05-4F20-A45C-8509CCA4E2D5}.Release|Any CPU.ActiveCfg = Release|Any CPU {71FA7684-9E05-4F20-A45C-8509CCA4E2D5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D02DF3C7-18AB-4D69-8819-6E4A21DC620C} EndGlobalSection EndGlobal