Repository: provectus/kafka-ui Branch: master Commit: 83b5a60cc085 Files: 1068 Total size: 2.9 MB Directory structure: gitextract_kz4bsw12/ ├── .devcontainer/ │ └── devcontainer.json ├── .editorconfig ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug.yml │ │ ├── config.yml │ │ └── feature.yml │ ├── PULL_REQUEST_TEMPLATE.md │ ├── dependabot.yml │ ├── release_drafter.yaml │ └── workflows/ │ ├── aws_publisher.yaml │ ├── backend.yml │ ├── block_merge.yml │ ├── branch-deploy.yml │ ├── branch-remove.yml │ ├── build-public-image.yml │ ├── codeql-analysis.yml │ ├── cve.yaml │ ├── delete-public-image.yml │ ├── documentation.yaml │ ├── e2e-automation.yml │ ├── e2e-checks.yaml │ ├── e2e-manual.yml │ ├── e2e-weekly.yml │ ├── frontend.yaml │ ├── master.yaml │ ├── pr-checks.yaml │ ├── release-serde-api.yaml │ ├── release.yaml │ ├── release_drafter.yml │ ├── separate_env_public_create.yml │ ├── separate_env_public_remove.yml │ ├── stale.yaml │ ├── terraform-deploy.yml │ ├── triage_issues.yml │ ├── triage_prs.yml │ ├── welcome-first-time-contributors.yml │ └── workflow_linter.yaml ├── .gitignore ├── .mvn/ │ └── wrapper/ │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── documentation/ │ └── compose/ │ ├── DOCKER_COMPOSE.md │ ├── connectors/ │ │ ├── github-source.json │ │ ├── s3-sink.json │ │ ├── sink-activities.json │ │ ├── source-activities.json │ │ └── start.sh │ ├── data/ │ │ ├── message.json │ │ └── proxy.conf │ ├── e2e-tests.yaml │ ├── jaas/ │ │ ├── client.properties │ │ ├── kafka_connect.jaas │ │ ├── kafka_connect.password │ │ ├── kafka_server.conf │ │ ├── schema_registry.jaas │ │ ├── schema_registry.password │ │ └── zookeeper_jaas.conf │ ├── jmx/ │ │ ├── clientkeystore │ │ ├── clienttruststore │ │ ├── jmxremote.access │ │ ├── jmxremote.password │ │ ├── serverkeystore │ │ └── servertruststore │ ├── jmx-exporter/ │ │ ├── kafka-broker.yml │ │ └── kafka-prepare-and-run │ ├── kafka-cluster-sr-auth.yaml │ ├── kafka-connect/ │ │ └── Dockerfile │ ├── kafka-ssl-components.yaml │ ├── kafka-ssl.yml │ ├── kafka-ui-acl-with-zk.yaml │ ├── kafka-ui-arm64.yaml │ ├── kafka-ui-auth-context.yaml │ ├── kafka-ui-connectors-auth.yaml │ ├── kafka-ui-jmx-secured.yml │ ├── kafka-ui-sasl.yaml │ ├── kafka-ui-serdes.yaml │ ├── kafka-ui-with-jmx-exporter.yaml │ ├── kafka-ui.yaml │ ├── kafka-with-zookeeper.yaml │ ├── ldap.yaml │ ├── nginx-proxy.yaml │ ├── postgres/ │ │ ├── Dockerfile │ │ └── data.sql │ ├── proto/ │ │ ├── key-types.proto │ │ └── values.proto │ ├── scripts/ │ │ ├── clusterID │ │ ├── create_cluster_id.sh │ │ ├── update_run.sh │ │ └── update_run_cluster.sh │ ├── ssl/ │ │ ├── creds │ │ ├── generate_certs.sh │ │ ├── kafka.keystore.jks │ │ ├── kafka.truststore.jks │ │ └── san.cnf │ ├── traefik/ │ │ └── kafkaui.yaml │ └── traefik-proxy.yaml ├── etc/ │ └── checkstyle/ │ ├── apache-header.txt │ ├── checkstyle-e2e.xml │ └── checkstyle.xml ├── kafka-ui-api/ │ ├── Dockerfile │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── antlr4/ │ │ │ └── ksql/ │ │ │ └── KsqlGrammar.g4 │ │ ├── java/ │ │ │ └── com/ │ │ │ └── provectus/ │ │ │ └── kafka/ │ │ │ └── ui/ │ │ │ ├── KafkaUiApplication.java │ │ │ ├── client/ │ │ │ │ └── RetryingKafkaConnectClient.java │ │ │ ├── config/ │ │ │ │ ├── ClustersProperties.java │ │ │ │ ├── Config.java │ │ │ │ ├── CorsGlobalConfiguration.java │ │ │ │ ├── CustomWebFilter.java │ │ │ │ ├── ReadOnlyModeFilter.java │ │ │ │ ├── WebclientProperties.java │ │ │ │ └── auth/ │ │ │ │ ├── AbstractAuthSecurityConfig.java │ │ │ │ ├── AuthenticatedUser.java │ │ │ │ ├── BasicAuthSecurityConfig.java │ │ │ │ ├── DisabledAuthSecurityConfig.java │ │ │ │ ├── LdapProperties.java │ │ │ │ ├── LdapSecurityConfig.java │ │ │ │ ├── OAuthProperties.java │ │ │ │ ├── OAuthPropertiesConverter.java │ │ │ │ ├── OAuthSecurityConfig.java │ │ │ │ ├── RbacLdapUser.java │ │ │ │ ├── RbacOAuth2User.java │ │ │ │ ├── RbacOidcUser.java │ │ │ │ ├── RbacUser.java │ │ │ │ ├── RoleBasedAccessControlProperties.java │ │ │ │ ├── condition/ │ │ │ │ │ ├── ActiveDirectoryCondition.java │ │ │ │ │ └── CognitoCondition.java │ │ │ │ └── logout/ │ │ │ │ ├── CognitoLogoutSuccessHandler.java │ │ │ │ ├── LogoutSuccessHandler.java │ │ │ │ └── OAuthLogoutSuccessHandler.java │ │ │ ├── controller/ │ │ │ │ ├── AbstractController.java │ │ │ │ ├── AccessController.java │ │ │ │ ├── AclsController.java │ │ │ │ ├── ApplicationConfigController.java │ │ │ │ ├── AuthController.java │ │ │ │ ├── BrokersController.java │ │ │ │ ├── ClustersController.java │ │ │ │ ├── ConsumerGroupsController.java │ │ │ │ ├── KafkaConnectController.java │ │ │ │ ├── KsqlController.java │ │ │ │ ├── MessagesController.java │ │ │ │ ├── SchemasController.java │ │ │ │ ├── StaticController.java │ │ │ │ └── TopicsController.java │ │ │ ├── emitter/ │ │ │ │ ├── AbstractEmitter.java │ │ │ │ ├── BackwardEmitter.java │ │ │ │ ├── ConsumingStats.java │ │ │ │ ├── EnhancedConsumer.java │ │ │ │ ├── ForwardEmitter.java │ │ │ │ ├── MessageFilters.java │ │ │ │ ├── MessagesProcessing.java │ │ │ │ ├── OffsetsInfo.java │ │ │ │ ├── PolledRecords.java │ │ │ │ ├── PollingSettings.java │ │ │ │ ├── PollingThrottler.java │ │ │ │ ├── RangePollingEmitter.java │ │ │ │ ├── ResultSizeLimiter.java │ │ │ │ ├── SeekOperations.java │ │ │ │ └── TailingEmitter.java │ │ │ ├── exception/ │ │ │ │ ├── ClusterNotFoundException.java │ │ │ │ ├── ConnectNotFoundException.java │ │ │ │ ├── CustomBaseException.java │ │ │ │ ├── DuplicateEntityException.java │ │ │ │ ├── ErrorCode.java │ │ │ │ ├── FileUploadException.java │ │ │ │ ├── GlobalErrorWebExceptionHandler.java │ │ │ │ ├── IllegalEntityStateException.java │ │ │ │ ├── InvalidRequestApiException.java │ │ │ │ ├── JsonAvroConversionException.java │ │ │ │ ├── KafkaConnectConflictReponseException.java │ │ │ │ ├── KsqlApiException.java │ │ │ │ ├── KsqlDbNotFoundException.java │ │ │ │ ├── LogDirNotFoundApiException.java │ │ │ │ ├── NotFoundException.java │ │ │ │ ├── ReadOnlyModeException.java │ │ │ │ ├── SchemaCompatibilityException.java │ │ │ │ ├── SchemaFailedToDeleteException.java │ │ │ │ ├── SchemaNotFoundException.java │ │ │ │ ├── TopicAnalysisException.java │ │ │ │ ├── TopicMetadataException.java │ │ │ │ ├── TopicNotFoundException.java │ │ │ │ ├── TopicOrPartitionNotFoundException.java │ │ │ │ ├── TopicRecreationException.java │ │ │ │ ├── UnprocessableEntityException.java │ │ │ │ └── ValidationException.java │ │ │ ├── mapper/ │ │ │ │ ├── ClusterMapper.java │ │ │ │ ├── ConsumerGroupMapper.java │ │ │ │ ├── DescribeLogDirsMapper.java │ │ │ │ ├── KafkaConnectMapper.java │ │ │ │ └── KafkaSrMapper.java │ │ │ ├── model/ │ │ │ │ ├── BrokerMetrics.java │ │ │ │ ├── CleanupPolicy.java │ │ │ │ ├── ClusterFeature.java │ │ │ │ ├── ConsumerPosition.java │ │ │ │ ├── InternalBroker.java │ │ │ │ ├── InternalBrokerConfig.java │ │ │ │ ├── InternalBrokerDiskUsage.java │ │ │ │ ├── InternalClusterMetrics.java │ │ │ │ ├── InternalClusterState.java │ │ │ │ ├── InternalConsumerGroup.java │ │ │ │ ├── InternalLogDirStats.java │ │ │ │ ├── InternalPartition.java │ │ │ │ ├── InternalPartitionsOffsets.java │ │ │ │ ├── InternalReplica.java │ │ │ │ ├── InternalSegmentSizeDto.java │ │ │ │ ├── InternalTopic.java │ │ │ │ ├── InternalTopicConfig.java │ │ │ │ ├── InternalTopicConsumerGroup.java │ │ │ │ ├── KafkaCluster.java │ │ │ │ ├── Metrics.java │ │ │ │ ├── MetricsConfig.java │ │ │ │ ├── PartitionDistributionStats.java │ │ │ │ ├── PartitionsStats.java │ │ │ │ ├── Statistics.java │ │ │ │ ├── connect/ │ │ │ │ │ └── InternalConnectInfo.java │ │ │ │ ├── rbac/ │ │ │ │ │ ├── AccessContext.java │ │ │ │ │ ├── Permission.java │ │ │ │ │ ├── Resource.java │ │ │ │ │ ├── Role.java │ │ │ │ │ ├── Subject.java │ │ │ │ │ ├── permission/ │ │ │ │ │ │ ├── AclAction.java │ │ │ │ │ │ ├── ApplicationConfigAction.java │ │ │ │ │ │ ├── AuditAction.java │ │ │ │ │ │ ├── ClusterConfigAction.java │ │ │ │ │ │ ├── ConnectAction.java │ │ │ │ │ │ ├── ConsumerGroupAction.java │ │ │ │ │ │ ├── KsqlAction.java │ │ │ │ │ │ ├── PermissibleAction.java │ │ │ │ │ │ ├── SchemaAction.java │ │ │ │ │ │ └── TopicAction.java │ │ │ │ │ └── provider/ │ │ │ │ │ └── Provider.java │ │ │ │ └── schemaregistry/ │ │ │ │ ├── ErrorResponse.java │ │ │ │ ├── InternalCompatibilityCheck.java │ │ │ │ ├── InternalCompatibilityLevel.java │ │ │ │ ├── InternalNewSchema.java │ │ │ │ └── SubjectIdResponse.java │ │ │ ├── serdes/ │ │ │ │ ├── BuiltInSerde.java │ │ │ │ ├── ClassloaderUtil.java │ │ │ │ ├── ClusterSerdes.java │ │ │ │ ├── ConsumerRecordDeserializer.java │ │ │ │ ├── CustomSerdeLoader.java │ │ │ │ ├── ProducerRecordCreator.java │ │ │ │ ├── PropertyResolverImpl.java │ │ │ │ ├── RecordHeaderImpl.java │ │ │ │ ├── RecordHeadersImpl.java │ │ │ │ ├── SerdeInstance.java │ │ │ │ ├── SerdesInitializer.java │ │ │ │ └── builtin/ │ │ │ │ ├── AvroEmbeddedSerde.java │ │ │ │ ├── Base64Serde.java │ │ │ │ ├── ConsumerOffsetsSerde.java │ │ │ │ ├── HexSerde.java │ │ │ │ ├── Int32Serde.java │ │ │ │ ├── Int64Serde.java │ │ │ │ ├── ProtobufFileSerde.java │ │ │ │ ├── ProtobufRawSerde.java │ │ │ │ ├── StringSerde.java │ │ │ │ ├── UInt32Serde.java │ │ │ │ ├── UInt64Serde.java │ │ │ │ ├── UuidBinarySerde.java │ │ │ │ └── sr/ │ │ │ │ ├── MessageFormatter.java │ │ │ │ ├── SchemaRegistrySerde.java │ │ │ │ ├── SchemaType.java │ │ │ │ └── Serialize.java │ │ │ ├── service/ │ │ │ │ ├── AdminClientService.java │ │ │ │ ├── AdminClientServiceImpl.java │ │ │ │ ├── ApplicationInfoService.java │ │ │ │ ├── BrokerService.java │ │ │ │ ├── ClusterService.java │ │ │ │ ├── ClustersStatisticsScheduler.java │ │ │ │ ├── ClustersStorage.java │ │ │ │ ├── ConsumerGroupService.java │ │ │ │ ├── DeserializationService.java │ │ │ │ ├── FeatureService.java │ │ │ │ ├── KafkaClusterFactory.java │ │ │ │ ├── KafkaConfigSanitizer.java │ │ │ │ ├── KafkaConnectService.java │ │ │ │ ├── MessagesService.java │ │ │ │ ├── OffsetsResetService.java │ │ │ │ ├── ReactiveAdminClient.java │ │ │ │ ├── SchemaRegistryService.java │ │ │ │ ├── StatisticsCache.java │ │ │ │ ├── StatisticsService.java │ │ │ │ ├── TopicsService.java │ │ │ │ ├── acl/ │ │ │ │ │ ├── AclCsv.java │ │ │ │ │ └── AclsService.java │ │ │ │ ├── analyze/ │ │ │ │ │ ├── AnalysisTasksStore.java │ │ │ │ │ ├── TopicAnalysisService.java │ │ │ │ │ ├── TopicAnalysisStats.java │ │ │ │ │ └── TopicIdentity.java │ │ │ │ ├── audit/ │ │ │ │ │ ├── AuditRecord.java │ │ │ │ │ ├── AuditService.java │ │ │ │ │ └── AuditWriter.java │ │ │ │ ├── integration/ │ │ │ │ │ └── odd/ │ │ │ │ │ ├── ConnectorInfo.java │ │ │ │ │ ├── ConnectorsExporter.java │ │ │ │ │ ├── OddExporter.java │ │ │ │ │ ├── OddExporterScheduler.java │ │ │ │ │ ├── OddIntegrationConfig.java │ │ │ │ │ ├── OddIntegrationProperties.java │ │ │ │ │ ├── Oddrn.java │ │ │ │ │ ├── SchemaReferencesResolver.java │ │ │ │ │ ├── TopicsExporter.java │ │ │ │ │ └── schema/ │ │ │ │ │ ├── AvroExtractor.java │ │ │ │ │ ├── DataSetFieldsExtractors.java │ │ │ │ │ ├── JsonSchemaExtractor.java │ │ │ │ │ └── ProtoExtractor.java │ │ │ │ ├── ksql/ │ │ │ │ │ ├── KsqlApiClient.java │ │ │ │ │ ├── KsqlGrammar.java │ │ │ │ │ ├── KsqlServiceV2.java │ │ │ │ │ └── response/ │ │ │ │ │ ├── DynamicParser.java │ │ │ │ │ └── ResponseParser.java │ │ │ │ ├── masking/ │ │ │ │ │ ├── DataMasking.java │ │ │ │ │ └── policies/ │ │ │ │ │ ├── FieldsSelector.java │ │ │ │ │ ├── Mask.java │ │ │ │ │ ├── MaskingPolicy.java │ │ │ │ │ ├── Remove.java │ │ │ │ │ └── Replace.java │ │ │ │ ├── metrics/ │ │ │ │ │ ├── JmxMetricsFormatter.java │ │ │ │ │ ├── JmxMetricsRetriever.java │ │ │ │ │ ├── JmxSslSocketFactory.java │ │ │ │ │ ├── MetricsCollector.java │ │ │ │ │ ├── MetricsRetriever.java │ │ │ │ │ ├── PrometheusEndpointMetricsParser.java │ │ │ │ │ ├── PrometheusMetricsRetriever.java │ │ │ │ │ ├── RawMetric.java │ │ │ │ │ └── WellKnownMetrics.java │ │ │ │ └── rbac/ │ │ │ │ ├── AbstractProviderCondition.java │ │ │ │ ├── AccessControlService.java │ │ │ │ └── extractor/ │ │ │ │ ├── CognitoAuthorityExtractor.java │ │ │ │ ├── GithubAuthorityExtractor.java │ │ │ │ ├── GoogleAuthorityExtractor.java │ │ │ │ ├── OauthAuthorityExtractor.java │ │ │ │ ├── ProviderAuthorityExtractor.java │ │ │ │ └── RbacLdapAuthoritiesExtractor.java │ │ │ └── util/ │ │ │ ├── ApplicationMetrics.java │ │ │ ├── ApplicationRestarter.java │ │ │ ├── DynamicConfigOperations.java │ │ │ ├── EmptyRedirectStrategy.java │ │ │ ├── GithubReleaseInfo.java │ │ │ ├── KafkaServicesValidation.java │ │ │ ├── KafkaVersion.java │ │ │ ├── ReactiveFailover.java │ │ │ ├── ResourceUtil.java │ │ │ ├── SslPropertiesUtil.java │ │ │ ├── WebClientConfigurator.java │ │ │ ├── annotation/ │ │ │ │ └── KafkaClientInternalsDependant.java │ │ │ └── jsonschema/ │ │ │ ├── AnyFieldSchema.java │ │ │ ├── ArrayFieldSchema.java │ │ │ ├── AvroJsonSchemaConverter.java │ │ │ ├── EnumJsonType.java │ │ │ ├── FieldSchema.java │ │ │ ├── JsonAvroConversion.java │ │ │ ├── JsonSchema.java │ │ │ ├── JsonSchemaConverter.java │ │ │ ├── JsonType.java │ │ │ ├── MapFieldSchema.java │ │ │ ├── ObjectFieldSchema.java │ │ │ ├── OneOfFieldSchema.java │ │ │ ├── ProtobufSchemaConverter.java │ │ │ ├── RefFieldSchema.java │ │ │ ├── SimpleFieldSchema.java │ │ │ └── SimpleJsonType.java │ │ └── resources/ │ │ ├── application-local.yml │ │ ├── application.yml │ │ ├── banner.txt │ │ ├── logback-spring.xml │ │ └── static/ │ │ └── static/ │ │ └── css/ │ │ ├── bootstrap.min.css │ │ └── signin.css │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── provectus/ │ │ └── kafka/ │ │ └── ui/ │ │ ├── AbstractIntegrationTest.java │ │ ├── KafkaConnectServiceTests.java │ │ ├── KafkaConsumerGroupTests.java │ │ ├── KafkaConsumerTests.java │ │ ├── KafkaTopicCreateTests.java │ │ ├── ReadOnlyModeTests.java │ │ ├── SchemaRegistryServiceTests.java │ │ ├── config/ │ │ │ └── ClustersPropertiesTest.java │ │ ├── container/ │ │ │ ├── KafkaConnectContainer.java │ │ │ ├── KsqlDbContainer.java │ │ │ └── SchemaRegistryContainer.java │ │ ├── controller/ │ │ │ └── ApplicationConfigControllerTest.java │ │ ├── emitter/ │ │ │ ├── MessageFiltersTest.java │ │ │ ├── MessagesProcessingTest.java │ │ │ ├── OffsetsInfoTest.java │ │ │ ├── SeekOperationsTest.java │ │ │ └── TailingEmitterTest.java │ │ ├── model/ │ │ │ └── PartitionDistributionStatsTest.java │ │ ├── producer/ │ │ │ └── KafkaTestProducer.java │ │ ├── serdes/ │ │ │ ├── ConsumerRecordDeserializerTest.java │ │ │ ├── PropertyResolverImplTest.java │ │ │ ├── SerdesInitializerTest.java │ │ │ └── builtin/ │ │ │ ├── AvroEmbeddedSerdeTest.java │ │ │ ├── Base64SerdeTest.java │ │ │ ├── ConsumerOffsetsSerdeTest.java │ │ │ ├── HexSerdeTest.java │ │ │ ├── Int32SerdeTest.java │ │ │ ├── Int64SerdeTest.java │ │ │ ├── ProtobufFileSerdeTest.java │ │ │ ├── ProtobufRawSerdeTest.java │ │ │ ├── UInt32SerdeTest.java │ │ │ ├── UInt64SerdeTest.java │ │ │ ├── UuidBinarySerdeTest.java │ │ │ └── sr/ │ │ │ └── SchemaRegistrySerdeTest.java │ │ ├── service/ │ │ │ ├── BrokerServiceTest.java │ │ │ ├── ConfigTest.java │ │ │ ├── KafkaConfigSanitizerTest.java │ │ │ ├── LogDirsTest.java │ │ │ ├── MessagesServiceTest.java │ │ │ ├── OffsetsResetServiceTest.java │ │ │ ├── ReactiveAdminClientTest.java │ │ │ ├── RecordEmitterTest.java │ │ │ ├── SchemaRegistryPaginationTest.java │ │ │ ├── SendAndReadTests.java │ │ │ ├── TopicsServicePaginationTest.java │ │ │ ├── acl/ │ │ │ │ ├── AclCsvTest.java │ │ │ │ └── AclsServiceTest.java │ │ │ ├── analyze/ │ │ │ │ └── TopicAnalysisServiceTest.java │ │ │ ├── audit/ │ │ │ │ ├── AuditIntegrationTest.java │ │ │ │ ├── AuditServiceTest.java │ │ │ │ └── AuditWriterTest.java │ │ │ ├── integration/ │ │ │ │ └── odd/ │ │ │ │ ├── ConnectorsExporterTest.java │ │ │ │ ├── SchemaReferencesResolverTest.java │ │ │ │ ├── TopicsExporterTest.java │ │ │ │ └── schema/ │ │ │ │ ├── AvroExtractorTest.java │ │ │ │ ├── JsonSchemaExtractorTest.java │ │ │ │ └── ProtoExtractorTest.java │ │ │ ├── ksql/ │ │ │ │ ├── KsqlApiClientTest.java │ │ │ │ ├── KsqlServiceV2Test.java │ │ │ │ └── response/ │ │ │ │ └── ResponseParserTest.java │ │ │ ├── masking/ │ │ │ │ ├── DataMaskingTest.java │ │ │ │ └── policies/ │ │ │ │ ├── FieldsSelectorTest.java │ │ │ │ ├── MaskTest.java │ │ │ │ ├── RemoveTest.java │ │ │ │ └── ReplaceTest.java │ │ │ └── metrics/ │ │ │ ├── JmxMetricsFormatterTest.java │ │ │ ├── PrometheusEndpointMetricsParserTest.java │ │ │ ├── PrometheusMetricsRetrieverTest.java │ │ │ └── WellKnownMetricsTest.java │ │ └── util/ │ │ ├── AccessControlServiceMock.java │ │ ├── DynamicConfigOperationsTest.java │ │ ├── GithubReleaseInfoTest.java │ │ ├── PollingThrottlerTest.java │ │ ├── ReactiveFailoverTest.java │ │ └── jsonschema/ │ │ ├── AvroJsonSchemaConverterTest.java │ │ ├── JsonAvroConversionTest.java │ │ └── ProtobufSchemaConverterTest.java │ └── resources/ │ ├── application-test.yml │ ├── fileForUploadTest.txt │ └── protobuf-serde/ │ ├── address-book.proto │ ├── lang-description.proto │ ├── language/ │ │ └── language.proto │ └── sensor.proto ├── kafka-ui-contract/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── resources/ │ └── swagger/ │ ├── kafka-connect-api.yaml │ ├── kafka-sr-api.yaml │ └── kafka-ui-api.yaml ├── kafka-ui-e2e-checks/ │ ├── .gitignore │ ├── QASE.md │ ├── README.md │ ├── docker/ │ │ ├── selenoid-git.yaml │ │ └── selenoid-local.yaml │ ├── pom.xml │ ├── selenoid/ │ │ └── config/ │ │ ├── browsersGit.json │ │ └── browsersLocal.json │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── provectus/ │ │ │ └── kafka/ │ │ │ └── ui/ │ │ │ ├── models/ │ │ │ │ ├── Connector.java │ │ │ │ ├── Schema.java │ │ │ │ └── Topic.java │ │ │ ├── pages/ │ │ │ │ ├── BasePage.java │ │ │ │ ├── brokers/ │ │ │ │ │ ├── BrokersConfigTab.java │ │ │ │ │ ├── BrokersDetails.java │ │ │ │ │ └── BrokersList.java │ │ │ │ ├── connectors/ │ │ │ │ │ ├── ConnectorCreateForm.java │ │ │ │ │ ├── ConnectorDetails.java │ │ │ │ │ └── KafkaConnectList.java │ │ │ │ ├── consumers/ │ │ │ │ │ ├── ConsumersDetails.java │ │ │ │ │ └── ConsumersList.java │ │ │ │ ├── ksqldb/ │ │ │ │ │ ├── KsqlDbList.java │ │ │ │ │ ├── KsqlQueryForm.java │ │ │ │ │ ├── enums/ │ │ │ │ │ │ ├── KsqlMenuTabs.java │ │ │ │ │ │ └── KsqlQueryConfig.java │ │ │ │ │ └── models/ │ │ │ │ │ ├── Stream.java │ │ │ │ │ └── Table.java │ │ │ │ ├── panels/ │ │ │ │ │ ├── NaviSideBar.java │ │ │ │ │ ├── TopPanel.java │ │ │ │ │ └── enums/ │ │ │ │ │ └── MenuItem.java │ │ │ │ ├── schemas/ │ │ │ │ │ ├── SchemaCreateForm.java │ │ │ │ │ ├── SchemaDetails.java │ │ │ │ │ └── SchemaRegistryList.java │ │ │ │ └── topics/ │ │ │ │ ├── ProduceMessagePanel.java │ │ │ │ ├── TopicCreateEditForm.java │ │ │ │ ├── TopicDetails.java │ │ │ │ ├── TopicSettingsTab.java │ │ │ │ ├── TopicsList.java │ │ │ │ └── enums/ │ │ │ │ ├── CleanupPolicyValue.java │ │ │ │ ├── CustomParameterType.java │ │ │ │ ├── MaxSizeOnDisk.java │ │ │ │ └── TimeToRetain.java │ │ │ ├── services/ │ │ │ │ └── ApiService.java │ │ │ ├── settings/ │ │ │ │ ├── BaseSource.java │ │ │ │ ├── configs/ │ │ │ │ │ ├── Config.java │ │ │ │ │ └── Profiles.java │ │ │ │ ├── drivers/ │ │ │ │ │ └── WebDriver.java │ │ │ │ └── listeners/ │ │ │ │ ├── AllureListener.java │ │ │ │ ├── LoggerListener.java │ │ │ │ ├── QaseCreateListener.java │ │ │ │ └── QaseResultListener.java │ │ │ ├── utilities/ │ │ │ │ ├── FileUtils.java │ │ │ │ ├── StringUtils.java │ │ │ │ ├── TimeUtils.java │ │ │ │ ├── WebUtils.java │ │ │ │ └── qase/ │ │ │ │ ├── QaseSetup.java │ │ │ │ ├── annotations/ │ │ │ │ │ ├── Automation.java │ │ │ │ │ ├── Status.java │ │ │ │ │ └── Suite.java │ │ │ │ └── enums/ │ │ │ │ ├── State.java │ │ │ │ └── Status.java │ │ │ └── variables/ │ │ │ ├── Browser.java │ │ │ ├── Expected.java │ │ │ ├── Suite.java │ │ │ └── Url.java │ │ └── resources/ │ │ ├── allure.properties │ │ └── testData/ │ │ ├── connectors/ │ │ │ ├── config_for_create_connector.json │ │ │ ├── config_for_create_connector_via_api.json │ │ │ ├── config_for_update_connector.json │ │ │ └── delete_connector_config.json │ │ ├── schemas/ │ │ │ ├── schema_avro_for_update.json │ │ │ ├── schema_avro_value.json │ │ │ ├── schema_json_Value.json │ │ │ └── schema_protobuf_value.txt │ │ └── topics/ │ │ └── message_content_create_topic.json │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── provectus/ │ │ └── kafka/ │ │ └── ui/ │ │ ├── BaseTest.java │ │ ├── Facade.java │ │ ├── manualsuite/ │ │ │ ├── BaseManualTest.java │ │ │ ├── backlog/ │ │ │ │ ├── SanityBacklog.java │ │ │ │ └── SmokeBacklog.java │ │ │ └── suite/ │ │ │ ├── DataMaskingTest.java │ │ │ ├── RbacTest.java │ │ │ ├── TopicsTest.java │ │ │ └── WizardTest.java │ │ ├── qasesuite/ │ │ │ ├── BaseQaseTest.java │ │ │ └── Template.java │ │ ├── sanitysuite/ │ │ │ └── TopicsTest.java │ │ └── smokesuite/ │ │ ├── SmokeTest.java │ │ ├── brokers/ │ │ │ └── BrokersTest.java │ │ ├── connectors/ │ │ │ └── ConnectorsTest.java │ │ ├── ksqldb/ │ │ │ └── KsqlDbTest.java │ │ ├── schemas/ │ │ │ └── SchemasTest.java │ │ └── topics/ │ │ ├── MessagesTest.java │ │ └── TopicsTest.java │ └── resources/ │ ├── manual.xml │ ├── qase.xml │ ├── regression.xml │ ├── sanity.xml │ └── smoke.xml ├── kafka-ui-react-app/ │ ├── .editorconfig │ ├── .eslintignore │ ├── .eslintrc.json │ ├── .gitignore │ ├── .jest/ │ │ ├── cssTransform.js │ │ └── resolver.js │ ├── .nvmrc │ ├── .prettierrc │ ├── README.md │ ├── index.html │ ├── jest.config.ts │ ├── openapitools.json │ ├── package.json │ ├── public/ │ │ ├── manifest.json │ │ └── robots.txt │ ├── sonar-project.properties │ ├── src/ │ │ ├── components/ │ │ │ ├── ACLPage/ │ │ │ │ ├── ACLPage.tsx │ │ │ │ └── List/ │ │ │ │ ├── List.styled.ts │ │ │ │ ├── List.tsx │ │ │ │ └── __test__/ │ │ │ │ └── List.spec.tsx │ │ │ ├── App.styled.ts │ │ │ ├── App.tsx │ │ │ ├── Brokers/ │ │ │ │ ├── Broker/ │ │ │ │ │ ├── Broker.tsx │ │ │ │ │ ├── BrokerLogdir/ │ │ │ │ │ │ ├── BrokerLogdir.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── BrokerLogdir.spec.tsx │ │ │ │ │ ├── BrokerMetrics/ │ │ │ │ │ │ ├── BrokerMetrics.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── BrokerMetrics.spec.tsx │ │ │ │ │ ├── Configs/ │ │ │ │ │ │ ├── Configs.styled.ts │ │ │ │ │ │ ├── Configs.tsx │ │ │ │ │ │ ├── InputCell.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── Configs.spec.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── Broker.spec.tsx │ │ │ │ ├── Brokers.tsx │ │ │ │ ├── BrokersList/ │ │ │ │ │ ├── BrokersList.styled.ts │ │ │ │ │ ├── BrokersList.tsx │ │ │ │ │ ├── SkewHeader/ │ │ │ │ │ │ ├── SkewHeader.styled.ts │ │ │ │ │ │ └── SkewHeader.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── BrokersList.spec.tsx │ │ │ │ ├── __test__/ │ │ │ │ │ └── Brokers.spec.tsx │ │ │ │ └── utils/ │ │ │ │ ├── __test__/ │ │ │ │ │ ├── fixtures.ts │ │ │ │ │ └── getEditorText.spec.tsx │ │ │ │ └── getEditorText.ts │ │ │ ├── ClusterPage/ │ │ │ │ ├── ClusterConfigPage.tsx │ │ │ │ ├── ClusterPage.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── ClusterPage.spec.tsx │ │ │ ├── Connect/ │ │ │ │ ├── Connect.tsx │ │ │ │ ├── Details/ │ │ │ │ │ ├── Actions/ │ │ │ │ │ │ ├── Action.styled.ts │ │ │ │ │ │ ├── Actions.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── Actions.spec.tsx │ │ │ │ │ ├── Config/ │ │ │ │ │ │ ├── Config.styled.ts │ │ │ │ │ │ ├── Config.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── Config.spec.tsx │ │ │ │ │ ├── DetailsPage.tsx │ │ │ │ │ ├── Overview/ │ │ │ │ │ │ ├── Overview.tsx │ │ │ │ │ │ ├── __tests__/ │ │ │ │ │ │ │ ├── Overview.spec.tsx │ │ │ │ │ │ │ └── getTaskMetrics.spec.ts │ │ │ │ │ │ └── getTaskMetrics.ts │ │ │ │ │ ├── Tasks/ │ │ │ │ │ │ ├── ActionsCellTasks.tsx │ │ │ │ │ │ ├── Tasks.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── Tasks.spec.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── DetailsPage.spec.tsx │ │ │ │ ├── List/ │ │ │ │ │ ├── ActionsCell.tsx │ │ │ │ │ ├── List.styled.ts │ │ │ │ │ ├── List.tsx │ │ │ │ │ ├── ListPage.tsx │ │ │ │ │ ├── RunningTasksCell.tsx │ │ │ │ │ ├── TopicsCell.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ ├── List.spec.tsx │ │ │ │ │ └── ListPage.spec.tsx │ │ │ │ ├── New/ │ │ │ │ │ ├── New.styled.ts │ │ │ │ │ ├── New.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── New.spec.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── Connect.spec.tsx │ │ │ ├── ConsumerGroups/ │ │ │ │ ├── ConsumerGroups.tsx │ │ │ │ ├── Details/ │ │ │ │ │ ├── Details.tsx │ │ │ │ │ ├── ListItem.styled.ts │ │ │ │ │ ├── ListItem.tsx │ │ │ │ │ ├── ResetOffsets/ │ │ │ │ │ │ ├── Form.tsx │ │ │ │ │ │ ├── ResetOffsets.styled.ts │ │ │ │ │ │ └── ResetOffsets.tsx │ │ │ │ │ └── TopicContents/ │ │ │ │ │ ├── TopicContent.styled.ts │ │ │ │ │ ├── TopicContents.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── TopicContents.spec.tsx │ │ │ │ ├── List.tsx │ │ │ │ └── __test__/ │ │ │ │ └── ConsumerGroups.spec.tsx │ │ │ ├── Dashboard/ │ │ │ │ ├── ClusterName.tsx │ │ │ │ ├── ClusterTableActionsCell.tsx │ │ │ │ ├── Dashboard.styled.ts │ │ │ │ └── Dashboard.tsx │ │ │ ├── ErrorPage/ │ │ │ │ ├── ErrorPage.styled.ts │ │ │ │ ├── ErrorPage.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── ErrorPage.spec.tsx │ │ │ ├── KsqlDb/ │ │ │ │ ├── KsqlDb.tsx │ │ │ │ ├── Query/ │ │ │ │ │ ├── Query.tsx │ │ │ │ │ ├── QueryForm/ │ │ │ │ │ │ ├── QueryForm.styled.ts │ │ │ │ │ │ └── QueryForm.tsx │ │ │ │ │ └── renderer/ │ │ │ │ │ └── TableRenderer/ │ │ │ │ │ ├── TableRenderer.styled.tsx │ │ │ │ │ └── TableRenderer.tsx │ │ │ │ └── TableView.tsx │ │ │ ├── Nav/ │ │ │ │ ├── ClusterMenu.tsx │ │ │ │ ├── ClusterMenuItem.tsx │ │ │ │ ├── ClusterTab/ │ │ │ │ │ ├── ClusterTab.styled.ts │ │ │ │ │ ├── ClusterTab.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ ├── ClusterTab.spec.tsx │ │ │ │ │ └── ClusterTab.styled.spec.tsx │ │ │ │ ├── Nav.styled.ts │ │ │ │ ├── Nav.tsx │ │ │ │ └── __tests__/ │ │ │ │ ├── ClusterMenu.spec.tsx │ │ │ │ ├── ClusterMenuItem.spec.tsx │ │ │ │ └── Nav.spec.tsx │ │ │ ├── NavBar/ │ │ │ │ ├── NavBar.styled.ts │ │ │ │ ├── NavBar.tsx │ │ │ │ ├── UserInfo/ │ │ │ │ │ ├── UserInfo.styled.ts │ │ │ │ │ ├── UserInfo.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── UserInfo.spec.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── NavBar.spec.tsx │ │ │ ├── PageContainer/ │ │ │ │ ├── PageContainer.styled.ts │ │ │ │ ├── PageContainer.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── PageContainer.spec.tsx │ │ │ ├── Schemas/ │ │ │ │ ├── Details/ │ │ │ │ │ ├── Details.tsx │ │ │ │ │ ├── LatestVersion/ │ │ │ │ │ │ ├── LatestVersionItem.styled.tsx │ │ │ │ │ │ └── LatestVersionItem.tsx │ │ │ │ │ ├── SchemaVersion/ │ │ │ │ │ │ └── SchemaVersion.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ ├── Details.spec.tsx │ │ │ │ │ ├── LatestVersionItem.spec.tsx │ │ │ │ │ ├── SchemaVersion.spec.tsx │ │ │ │ │ └── fixtures.ts │ │ │ │ ├── Diff/ │ │ │ │ │ ├── Diff.styled.ts │ │ │ │ │ ├── Diff.tsx │ │ │ │ │ ├── DiffContainer.ts │ │ │ │ │ └── __test__/ │ │ │ │ │ ├── Diff.spec.tsx │ │ │ │ │ └── fixtures.ts │ │ │ │ ├── Edit/ │ │ │ │ │ ├── Edit.styled.ts │ │ │ │ │ ├── Edit.tsx │ │ │ │ │ ├── Form.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Edit.spec.tsx │ │ │ │ ├── List/ │ │ │ │ │ ├── GlobalSchemaSelector/ │ │ │ │ │ │ ├── GlobalSchemaSelector.styled.ts │ │ │ │ │ │ ├── GlobalSchemaSelector.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── GlobalSchemaSelector.spec.tsx │ │ │ │ │ ├── List.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ ├── List.spec.tsx │ │ │ │ │ └── fixtures.ts │ │ │ │ ├── New/ │ │ │ │ │ ├── New.styled.ts │ │ │ │ │ ├── New.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── New.spec.tsx │ │ │ │ ├── Schemas.tsx │ │ │ │ └── __test__/ │ │ │ │ └── Schemas.spec.tsx │ │ │ ├── Topics/ │ │ │ │ ├── List/ │ │ │ │ │ ├── ActionsCell.tsx │ │ │ │ │ ├── BatchActionsBar.tsx │ │ │ │ │ ├── ListPage.tsx │ │ │ │ │ ├── TopicTable.tsx │ │ │ │ │ ├── TopicTitleCell.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ ├── ListPage.spec.tsx │ │ │ │ │ └── TopicTable.spec.tsx │ │ │ │ ├── New/ │ │ │ │ │ ├── New.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── New.spec.tsx │ │ │ │ ├── Topic/ │ │ │ │ │ ├── ConsumerGroups/ │ │ │ │ │ │ ├── TopicConsumerGroups.styled.ts │ │ │ │ │ │ ├── TopicConsumerGroups.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── TopicConsumerGroups.spec.tsx │ │ │ │ │ ├── Edit/ │ │ │ │ │ │ ├── DangerZone/ │ │ │ │ │ │ │ ├── DangerZone.styled.tsx │ │ │ │ │ │ │ ├── DangerZone.tsx │ │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ │ └── DangerZone.spec.tsx │ │ │ │ │ │ ├── Edit.tsx │ │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ │ ├── Edit.spec.tsx │ │ │ │ │ │ │ └── topicParamsTransformer.spec.ts │ │ │ │ │ │ └── topicParamsTransformer.ts │ │ │ │ │ ├── Messages/ │ │ │ │ │ │ ├── Filters/ │ │ │ │ │ │ │ ├── AddEditFilterContainer.tsx │ │ │ │ │ │ │ ├── AddFilter.tsx │ │ │ │ │ │ │ ├── EditFilter.tsx │ │ │ │ │ │ │ ├── FilterModal.tsx │ │ │ │ │ │ │ ├── Filters.styled.ts │ │ │ │ │ │ │ ├── Filters.tsx │ │ │ │ │ │ │ ├── FiltersContainer.ts │ │ │ │ │ │ │ ├── InfoModal.tsx │ │ │ │ │ │ │ ├── SavedFilters.tsx │ │ │ │ │ │ │ ├── __tests__/ │ │ │ │ │ │ │ │ ├── AddEditFilterContainer.spec.tsx │ │ │ │ │ │ │ │ ├── AddFilter.spec.tsx │ │ │ │ │ │ │ │ ├── EditFilter.spec.tsx │ │ │ │ │ │ │ │ ├── FilterModal.spec.tsx │ │ │ │ │ │ │ │ ├── Filters.spec.tsx │ │ │ │ │ │ │ │ ├── Filters.styled.spec.tsx │ │ │ │ │ │ │ │ ├── InfoModal.spec.tsx │ │ │ │ │ │ │ │ └── SavedFilters.spec.tsx │ │ │ │ │ │ │ └── utils.ts │ │ │ │ │ │ ├── Message.tsx │ │ │ │ │ │ ├── MessageContent/ │ │ │ │ │ │ │ ├── MessageContent.styled.ts │ │ │ │ │ │ │ ├── MessageContent.tsx │ │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ │ └── MessageContent.spec.tsx │ │ │ │ │ │ ├── Messages.styled.ts │ │ │ │ │ │ ├── Messages.tsx │ │ │ │ │ │ ├── MessagesTable.tsx │ │ │ │ │ │ ├── PreviewModal.styled.ts │ │ │ │ │ │ ├── PreviewModal.tsx │ │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ │ ├── FiltersContainer.spec.tsx │ │ │ │ │ │ │ ├── Message.spec.tsx │ │ │ │ │ │ │ ├── Messages.spec.tsx │ │ │ │ │ │ │ ├── MessagesTable.spec.tsx │ │ │ │ │ │ │ ├── PreviewModal.spec.tsx │ │ │ │ │ │ │ └── utils.spec.ts │ │ │ │ │ │ └── getDefaultSerdeName.ts │ │ │ │ │ ├── Overview/ │ │ │ │ │ │ ├── ActionsCell.tsx │ │ │ │ │ │ ├── Overview.styled.ts │ │ │ │ │ │ ├── Overview.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── Overview.spec.tsx │ │ │ │ │ ├── SendMessage/ │ │ │ │ │ │ ├── SendMessage.styled.tsx │ │ │ │ │ │ ├── SendMessage.tsx │ │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ │ ├── SendMessage.spec.tsx │ │ │ │ │ │ │ └── utils.spec.ts │ │ │ │ │ │ └── utils.ts │ │ │ │ │ ├── Settings/ │ │ │ │ │ │ ├── Settings.styled.ts │ │ │ │ │ │ ├── Settings.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ └── Settings.spec.tsx │ │ │ │ │ ├── Statistics/ │ │ │ │ │ │ ├── Indicators/ │ │ │ │ │ │ │ ├── SizeStats.tsx │ │ │ │ │ │ │ └── Total.tsx │ │ │ │ │ │ ├── Metrics.tsx │ │ │ │ │ │ ├── PartitionInfoRow.tsx │ │ │ │ │ │ ├── PartitionTable.tsx │ │ │ │ │ │ ├── Statistics.styles.ts │ │ │ │ │ │ ├── Statistics.tsx │ │ │ │ │ │ └── __test__/ │ │ │ │ │ │ ├── Metrics.spec.tsx │ │ │ │ │ │ └── Statistics.spec.tsx │ │ │ │ │ ├── Topic.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── Topic.spec.tsx │ │ │ │ ├── Topics.tsx │ │ │ │ ├── __tests__/ │ │ │ │ │ └── Topics.spec.tsx │ │ │ │ └── shared/ │ │ │ │ └── Form/ │ │ │ │ ├── CustomParams/ │ │ │ │ │ ├── CustomParamField.tsx │ │ │ │ │ ├── CustomParams.styled.ts │ │ │ │ │ ├── CustomParams.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ ├── CustomParamField.spec.tsx │ │ │ │ │ ├── CustomParams.spec.tsx │ │ │ │ │ └── fixtures.ts │ │ │ │ ├── TimeToRetain.tsx │ │ │ │ ├── TimeToRetainBtn.tsx │ │ │ │ ├── TimeToRetainBtns.tsx │ │ │ │ ├── TopicForm.styled.ts │ │ │ │ ├── TopicForm.tsx │ │ │ │ └── __tests__/ │ │ │ │ ├── TimeToRetainBtn.spec.tsx │ │ │ │ ├── TimeToRetainBtns.spec.tsx │ │ │ │ ├── TopicForm.spec.tsx │ │ │ │ └── TopicForm.styled.spec.tsx │ │ │ ├── Version/ │ │ │ │ ├── Version.styled.ts │ │ │ │ ├── Version.tsx │ │ │ │ ├── __tests__/ │ │ │ │ │ ├── Version.spec.tsx │ │ │ │ │ └── compareVersions.spec.ts │ │ │ │ └── compareVersions.ts │ │ │ ├── __tests__/ │ │ │ │ └── App.spec.tsx │ │ │ ├── common/ │ │ │ │ ├── ActionComponent/ │ │ │ │ │ ├── ActionButton/ │ │ │ │ │ │ ├── ActionButton.tsx │ │ │ │ │ │ ├── ActionCanButton/ │ │ │ │ │ │ │ ├── ActionCanButton.tsx │ │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ │ └── ActionCanButton.spec.tsx │ │ │ │ │ │ ├── ActionCreateButton/ │ │ │ │ │ │ │ ├── ActionCreateButton.tsx │ │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ │ └── ActionCreateButton.spec.tsx │ │ │ │ │ │ ├── ActionPermissionButton/ │ │ │ │ │ │ │ ├── ActionPermissionButton.tsx │ │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ │ └── ActionPermissionButton.spec.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── ActionButton.spec.tsx │ │ │ │ │ ├── ActionComponent.styled.ts │ │ │ │ │ ├── ActionComponent.ts │ │ │ │ │ ├── ActionDropDownItem/ │ │ │ │ │ │ └── ActionDropdownItem.tsx │ │ │ │ │ ├── ActionNavLink/ │ │ │ │ │ │ ├── ActionNavLink.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── ActionNavLink.spec.tsx │ │ │ │ │ ├── ActionSelect/ │ │ │ │ │ │ ├── ActionSelect.tsx │ │ │ │ │ │ └── __tests__/ │ │ │ │ │ │ └── ActionSelect.spec.tsx │ │ │ │ │ ├── __tests__/ │ │ │ │ │ │ └── fixtures.ts │ │ │ │ │ └── index.ts │ │ │ │ ├── Alert/ │ │ │ │ │ ├── Alert.styled.ts │ │ │ │ │ ├── Alert.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Alert.spec.tsx │ │ │ │ ├── Button/ │ │ │ │ │ ├── Button.styled.ts │ │ │ │ │ ├── Button.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Button.spec.tsx │ │ │ │ ├── BytesFormatted/ │ │ │ │ │ ├── BytesFormatted.styled.ts │ │ │ │ │ ├── BytesFormatted.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── BytesFormatted.spec.tsx │ │ │ │ ├── Checkbox/ │ │ │ │ │ └── Checkbox.tsx │ │ │ │ ├── ConfirmationModal/ │ │ │ │ │ ├── ConfirmationModal.styled.tsx │ │ │ │ │ └── ConfirmationModal.tsx │ │ │ │ ├── ControlPanel/ │ │ │ │ │ └── ControlPanel.styled.ts │ │ │ │ ├── DiffViewer/ │ │ │ │ │ ├── DiffViewer.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── DiffViewer.spec.tsx │ │ │ │ ├── Dropdown/ │ │ │ │ │ ├── Dropdown.styled.ts │ │ │ │ │ ├── Dropdown.tsx │ │ │ │ │ ├── DropdownItem.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── Editor/ │ │ │ │ │ └── Editor.tsx │ │ │ │ ├── EditorViewer/ │ │ │ │ │ ├── EditorViewer.styled.ts │ │ │ │ │ ├── EditorViewer.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── EditorViewer.spec.tsx │ │ │ │ ├── Ellipsis/ │ │ │ │ │ ├── Ellipsis.styled.ts │ │ │ │ │ └── Ellipsis.tsx │ │ │ │ ├── Form/ │ │ │ │ │ └── Form.styled.ts │ │ │ │ ├── Icons/ │ │ │ │ │ ├── ArrowDownIcon.tsx │ │ │ │ │ ├── AutoIcon.tsx │ │ │ │ │ ├── CancelIcon.tsx │ │ │ │ │ ├── CheckMarkRoundIcon.tsx │ │ │ │ │ ├── CheckmarkIcon.tsx │ │ │ │ │ ├── ChevronDownIcon.tsx │ │ │ │ │ ├── ClockIcon.tsx │ │ │ │ │ ├── CloseCircleIcon.tsx │ │ │ │ │ ├── CloseIcon.tsx │ │ │ │ │ ├── DeleteIcon.tsx │ │ │ │ │ ├── DiscordIcon.tsx │ │ │ │ │ ├── DropdownArrowIcon.tsx │ │ │ │ │ ├── EditIcon.tsx │ │ │ │ │ ├── FileIcon.tsx │ │ │ │ │ ├── GitIcon.tsx │ │ │ │ │ ├── IconButtonWrapper.ts │ │ │ │ │ ├── InfoIcon.tsx │ │ │ │ │ ├── MessageToggleIcon.styled.ts │ │ │ │ │ ├── MessageToggleIcon.tsx │ │ │ │ │ ├── MoonIcon.tsx │ │ │ │ │ ├── PlusIcon.tsx │ │ │ │ │ ├── QuestionIcon.tsx │ │ │ │ │ ├── SavedIcon.tsx │ │ │ │ │ ├── SearchIcon.tsx │ │ │ │ │ ├── SpinnerIcon.tsx │ │ │ │ │ ├── StarIcon.tsx │ │ │ │ │ ├── SunIcon.tsx │ │ │ │ │ ├── UserIcon.tsx │ │ │ │ │ ├── VerticalElipsisIcon.tsx │ │ │ │ │ ├── WarningIcon.tsx │ │ │ │ │ └── WarningRedIcon.tsx │ │ │ │ ├── IndeterminateCheckbox/ │ │ │ │ │ └── IndeterminateCheckbox.tsx │ │ │ │ ├── Input/ │ │ │ │ │ ├── Input.styled.ts │ │ │ │ │ ├── Input.tsx │ │ │ │ │ ├── InputLabel.styled.ts │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Input.spec.tsx │ │ │ │ ├── Logo/ │ │ │ │ │ └── Logo.tsx │ │ │ │ ├── Metrics/ │ │ │ │ │ ├── Indicator.tsx │ │ │ │ │ ├── Metrics.styled.tsx │ │ │ │ │ ├── Section.tsx │ │ │ │ │ ├── __tests__/ │ │ │ │ │ │ ├── Indicator.spec.tsx │ │ │ │ │ │ └── Section.spec.tsx │ │ │ │ │ └── index.tsx │ │ │ │ ├── MultiSelect/ │ │ │ │ │ └── MultiSelect.styled.ts │ │ │ │ ├── Navigation/ │ │ │ │ │ └── Navbar.styled.ts │ │ │ │ ├── NewTable/ │ │ │ │ │ ├── ColoredCell.tsx │ │ │ │ │ ├── ExpanderCell.tsx │ │ │ │ │ ├── LinkCell.tsx │ │ │ │ │ ├── SelectRowCell.tsx │ │ │ │ │ ├── SelectRowHeader.tsx │ │ │ │ │ ├── SizeCell.tsx │ │ │ │ │ ├── Table.styled.ts │ │ │ │ │ ├── Table.tsx │ │ │ │ │ ├── TagCell.tsx │ │ │ │ │ ├── TimestampCell.tsx │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ └── Table.spec.tsx │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils/ │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ └── updateSortingState.spec.ts │ │ │ │ │ ├── updatePaginationState.ts │ │ │ │ │ └── updateSortingState.ts │ │ │ │ ├── PageHeading/ │ │ │ │ │ ├── PageHeading.styled.ts │ │ │ │ │ └── PageHeading.tsx │ │ │ │ ├── PageLoader/ │ │ │ │ │ ├── PageLoader.styled.ts │ │ │ │ │ ├── PageLoader.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── PageLoader.spec.tsx │ │ │ │ ├── ProgressBar/ │ │ │ │ │ ├── ProgressBar.styled.ts │ │ │ │ │ ├── ProgressBar.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── ProgressBar.spec.tsx │ │ │ │ ├── PropertiesList/ │ │ │ │ │ └── PropertiesList.styled.tsx │ │ │ │ ├── SQLEditor/ │ │ │ │ │ ├── SQLEditor.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── SQLEditor.spec.tsx │ │ │ │ ├── Search/ │ │ │ │ │ ├── Search.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Search.spec.tsx │ │ │ │ ├── Select/ │ │ │ │ │ ├── ControlledSelect.tsx │ │ │ │ │ ├── LiveIcon.styled.tsx │ │ │ │ │ ├── Select.styled.ts │ │ │ │ │ ├── Select.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Select.spec.tsx │ │ │ │ ├── SlidingSidebar/ │ │ │ │ │ ├── SlidingSidebar.styled.ts │ │ │ │ │ ├── SlidingSidebar.tsx │ │ │ │ │ └── index.ts │ │ │ │ ├── Spinner/ │ │ │ │ │ ├── Spinner.styled.ts │ │ │ │ │ ├── Spinner.tsx │ │ │ │ │ └── types.ts │ │ │ │ ├── SuspenseQueryComponent/ │ │ │ │ │ ├── SuspenseQueryComponent.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── SuspenseQueryComponent.spec.tsx │ │ │ │ ├── Switch/ │ │ │ │ │ ├── Switch.styled.ts │ │ │ │ │ └── Switch.tsx │ │ │ │ ├── Tag/ │ │ │ │ │ ├── Tag.styled.tsx │ │ │ │ │ └── getTagColor.ts │ │ │ │ ├── Textbox/ │ │ │ │ │ └── Textarea.styled.ts │ │ │ │ ├── Tooltip/ │ │ │ │ │ ├── Tooltip.styled.ts │ │ │ │ │ ├── Tooltip.tsx │ │ │ │ │ └── __tests__/ │ │ │ │ │ └── Tooltip.spec.tsx │ │ │ │ ├── heading/ │ │ │ │ │ └── Heading.styled.tsx │ │ │ │ └── table/ │ │ │ │ ├── Table/ │ │ │ │ │ ├── Table.styled.ts │ │ │ │ │ └── TableKeyLink.styled.ts │ │ │ │ ├── TableHeaderCell/ │ │ │ │ │ ├── TableHeaderCell.styled.ts │ │ │ │ │ ├── TableHeaderCell.tsx │ │ │ │ │ └── __test__/ │ │ │ │ │ └── TableHeaderCell.styled.spec.tsx │ │ │ │ ├── TableTitle/ │ │ │ │ │ └── TableTitle.styled.tsx │ │ │ │ └── __tests__/ │ │ │ │ └── TableHeaderCell.spec.tsx │ │ │ ├── contexts/ │ │ │ │ ├── ClusterContext.ts │ │ │ │ ├── ConfirmContext.tsx │ │ │ │ ├── GlobalSettingsContext.tsx │ │ │ │ ├── ThemeModeContext.tsx │ │ │ │ ├── TopicMessagesContext.ts │ │ │ │ └── UserInfoRolesAccessContext.tsx │ │ │ └── globalCss.ts │ │ ├── index.tsx │ │ ├── lib/ │ │ │ ├── __test__/ │ │ │ │ ├── dateTimeHelpers.spec.ts │ │ │ │ ├── paths.spec.ts │ │ │ │ ├── permission.spec.ts │ │ │ │ └── yupExtended.spec.ts │ │ │ ├── api.ts │ │ │ ├── constants.ts │ │ │ ├── dateTimeHelpers.ts │ │ │ ├── errorHandling.tsx │ │ │ ├── fixtures/ │ │ │ │ ├── acls.ts │ │ │ │ ├── brokers.ts │ │ │ │ ├── clusters.ts │ │ │ │ ├── consumerGroups.ts │ │ │ │ ├── kafkaConnect.ts │ │ │ │ ├── latestVersion.ts │ │ │ │ ├── topicMessages.ts │ │ │ │ └── topics.ts │ │ │ ├── hooks/ │ │ │ │ ├── __tests__/ │ │ │ │ │ ├── dateTimeHelpers.spec.ts │ │ │ │ │ ├── fixtures.ts │ │ │ │ │ ├── useBoolean.spec.ts │ │ │ │ │ ├── useCreatePermission.spec.tsx │ │ │ │ │ ├── useDataSaver.spec.tsx │ │ │ │ │ └── usePermission.spec.tsx │ │ │ │ ├── api/ │ │ │ │ │ ├── __tests__/ │ │ │ │ │ │ ├── brokers.spec.ts │ │ │ │ │ │ ├── clusters.spec.ts │ │ │ │ │ │ ├── kafkaConnect.spec.ts │ │ │ │ │ │ ├── latestVersion.spec.ts │ │ │ │ │ │ ├── topicMessages.spec.ts │ │ │ │ │ │ └── topics.spec.ts │ │ │ │ │ ├── acl.ts │ │ │ │ │ ├── appConfig.ts │ │ │ │ │ ├── brokers.ts │ │ │ │ │ ├── clusters.ts │ │ │ │ │ ├── consumers.ts │ │ │ │ │ ├── kafkaConnect.ts │ │ │ │ │ ├── ksqlDb.tsx │ │ │ │ │ ├── latestVersion.ts │ │ │ │ │ ├── roles.ts │ │ │ │ │ ├── topicMessages.tsx │ │ │ │ │ └── topics.ts │ │ │ │ ├── redux.ts │ │ │ │ ├── useActionTooltip.ts │ │ │ │ ├── useAppParams.tsx │ │ │ │ ├── useBoolean.ts │ │ │ │ ├── useClickOutside.ts │ │ │ │ ├── useConfirm.ts │ │ │ │ ├── useCreatePermisson.ts │ │ │ │ ├── useDataSaver.ts │ │ │ │ ├── useLocalStorage.ts │ │ │ │ ├── useMessageFiltersStore.ts │ │ │ │ ├── usePermission.ts │ │ │ │ └── useUserInfo.ts │ │ │ ├── paths.ts │ │ │ ├── permissions.ts │ │ │ ├── testHelpers.tsx │ │ │ └── yupExtended.ts │ │ ├── react-app-env.d.ts │ │ ├── redux/ │ │ │ ├── interfaces/ │ │ │ │ ├── cluster.ts │ │ │ │ ├── consumerGroup.ts │ │ │ │ ├── index.ts │ │ │ │ ├── loader.ts │ │ │ │ ├── schema.ts │ │ │ │ └── topic.ts │ │ │ ├── reducers/ │ │ │ │ ├── index.ts │ │ │ │ ├── loader/ │ │ │ │ │ ├── loaderSlice.ts │ │ │ │ │ └── selectors.ts │ │ │ │ ├── schemas/ │ │ │ │ │ ├── __test__/ │ │ │ │ │ │ └── fixtures.ts │ │ │ │ │ └── schemasSlice.ts │ │ │ │ └── topicMessages/ │ │ │ │ ├── __test__/ │ │ │ │ │ ├── fixtures.ts │ │ │ │ │ ├── reducer.spec.ts │ │ │ │ │ └── selectors.spec.ts │ │ │ │ ├── selectors.ts │ │ │ │ └── topicMessagesSlice.ts │ │ │ └── store/ │ │ │ └── index.ts │ │ ├── setupTests.ts │ │ ├── styled.d.ts │ │ ├── theme/ │ │ │ ├── index.scss │ │ │ ├── minireset.css │ │ │ └── theme.ts │ │ └── widgets/ │ │ └── ClusterConfigForm/ │ │ ├── ClusterConfigForm.styled.ts │ │ ├── Sections/ │ │ │ ├── Authentication/ │ │ │ │ ├── Authentication.tsx │ │ │ │ └── AuthenticationMethods.tsx │ │ │ ├── CustomAuthentication.tsx │ │ │ ├── KSQL.tsx │ │ │ ├── KafkaCluster.tsx │ │ │ ├── KafkaConnect.tsx │ │ │ ├── Metrics.tsx │ │ │ └── SchemaRegistry.tsx │ │ ├── common/ │ │ │ ├── Credentials.tsx │ │ │ ├── Fileupload.tsx │ │ │ ├── SSLForm.tsx │ │ │ └── SectionHeader.tsx │ │ ├── index.tsx │ │ ├── schema.ts │ │ ├── types.ts │ │ └── utils/ │ │ ├── convertFormKeyToPropsKey.ts │ │ ├── convertPropsKeyToFormKey.ts │ │ ├── getInitialFormData.ts │ │ ├── getIsValidConfig.ts │ │ ├── getJaasConfig.ts │ │ └── transformFormDataToPayload.ts │ ├── tsconfig.dev.json │ ├── tsconfig.json │ └── vite.config.ts ├── kafka-ui-serde-api/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── provectus/ │ └── kafka/ │ └── ui/ │ └── serde/ │ └── api/ │ ├── DeserializeResult.java │ ├── PropertyResolver.java │ ├── RecordHeader.java │ ├── RecordHeaders.java │ ├── SchemaDescription.java │ └── Serde.java ├── mvnw ├── mvnw.cmd ├── pom.xml └── settings.xml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .devcontainer/devcontainer.json ================================================ { "name": "Java", "image": "mcr.microsoft.com/devcontainers/java:0-17", "features": { "ghcr.io/devcontainers/features/java:1": { "version": "none", "installMaven": "true", "installGradle": "false" }, "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "java -version", "customizations": { "vscode": { "extensions" : [ "vscjava.vscode-java-pack", "vscjava.vscode-maven", "vscjava.vscode-java-debug", "EditorConfig.EditorConfig", "ms-azuretools.vscode-docker", "antfu.vite", "ms-kubernetes-tools.vscode-kubernetes-tools", "github.vscode-pull-request-github" ] } } } ================================================ FILE: .editorconfig ================================================ [*] charset = utf-8 end_of_line = lf indent_size = 4 indent_style = space insert_final_newline = true max_line_length = 120 tab_width = 4 ij_continuation_indent_size = 8 ij_formatter_off_tag = @formatter:off ij_formatter_on_tag = @formatter:on ij_formatter_tags_enabled = true ij_smart_tabs = false ij_visual_guides = none ij_wrap_on_typing = false trim_trailing_whitespace = true [*.java] indent_size = 2 ij_continuation_indent_size = 4 ij_java_align_consecutive_assignments = false ij_java_align_consecutive_variable_declarations = false ij_java_align_group_field_declarations = false ij_java_align_multiline_annotation_parameters = false ij_java_align_multiline_array_initializer_expression = false ij_java_align_multiline_assignment = false ij_java_align_multiline_binary_operation = false ij_java_align_multiline_chained_methods = false ij_java_align_multiline_extends_list = false ij_java_align_multiline_for = true ij_java_align_multiline_method_parentheses = false ij_java_align_multiline_parameters = true ij_java_align_multiline_parameters_in_calls = false ij_java_align_multiline_parenthesized_expression = false ij_java_align_multiline_records = true ij_java_align_multiline_resources = true ij_java_align_multiline_ternary_operation = false ij_java_align_multiline_text_blocks = false ij_java_align_multiline_throws_list = false ij_java_align_subsequent_simple_methods = false ij_java_align_throws_keyword = false ij_java_align_types_in_multi_catch = true ij_java_annotation_parameter_wrap = off ij_java_array_initializer_new_line_after_left_brace = false ij_java_array_initializer_right_brace_on_new_line = false ij_java_array_initializer_wrap = normal ij_java_assert_statement_colon_on_next_line = false ij_java_assert_statement_wrap = normal ij_java_assignment_wrap = normal ij_java_binary_operation_sign_on_next_line = false ij_java_binary_operation_wrap = normal ij_java_blank_lines_after_anonymous_class_header = 0 ij_java_blank_lines_after_class_header = 0 ij_java_blank_lines_after_imports = 1 ij_java_blank_lines_after_package = 1 ij_java_blank_lines_around_class = 1 ij_java_blank_lines_around_field = 0 ij_java_blank_lines_around_field_in_interface = 0 ij_java_blank_lines_around_initializer = 1 ij_java_blank_lines_around_method = 1 ij_java_blank_lines_around_method_in_interface = 1 ij_java_blank_lines_before_class_end = 0 ij_java_blank_lines_before_imports = 1 ij_java_blank_lines_before_method_body = 0 ij_java_blank_lines_before_package = 1 ij_java_block_brace_style = end_of_line ij_java_block_comment_add_space = false ij_java_block_comment_at_first_column = true ij_java_builder_methods = none ij_java_call_parameters_new_line_after_left_paren = false ij_java_call_parameters_right_paren_on_new_line = false ij_java_call_parameters_wrap = normal ij_java_case_statement_on_separate_line = true ij_java_catch_on_new_line = false ij_java_class_annotation_wrap = split_into_lines ij_java_class_brace_style = end_of_line ij_java_class_count_to_use_import_on_demand = 999 ij_java_class_names_in_javadoc = 1 ij_java_do_not_indent_top_level_class_members = false ij_java_do_not_wrap_after_single_annotation = false ij_java_do_not_wrap_after_single_annotation_in_parameter = false ij_java_do_while_brace_force = always ij_java_doc_add_blank_line_after_description = true ij_java_doc_add_blank_line_after_param_comments = false ij_java_doc_add_blank_line_after_return = false ij_java_doc_add_p_tag_on_empty_lines = true ij_java_doc_align_exception_comments = true ij_java_doc_align_param_comments = true ij_java_doc_do_not_wrap_if_one_line = false ij_java_doc_enable_formatting = true ij_java_doc_enable_leading_asterisks = true ij_java_doc_indent_on_continuation = false ij_java_doc_keep_empty_lines = true ij_java_doc_keep_empty_parameter_tag = true ij_java_doc_keep_empty_return_tag = true ij_java_doc_keep_empty_throws_tag = true ij_java_doc_keep_invalid_tags = true ij_java_doc_param_description_on_new_line = false ij_java_doc_preserve_line_breaks = false ij_java_doc_use_throws_not_exception_tag = true ij_java_else_on_new_line = false ij_java_entity_dd_suffix = EJB ij_java_entity_eb_suffix = Bean ij_java_entity_hi_suffix = Home ij_java_entity_lhi_prefix = Local ij_java_entity_lhi_suffix = Home ij_java_entity_li_prefix = Local ij_java_entity_pk_class = java.lang.String ij_java_entity_vo_suffix = VO ij_java_enum_constants_wrap = normal ij_java_extends_keyword_wrap = normal ij_java_extends_list_wrap = normal ij_java_field_annotation_wrap = split_into_lines ij_java_finally_on_new_line = false ij_java_for_brace_force = always ij_java_for_statement_new_line_after_left_paren = false ij_java_for_statement_right_paren_on_new_line = false ij_java_for_statement_wrap = normal ij_java_generate_final_locals = false ij_java_generate_final_parameters = false ij_java_if_brace_force = always ij_java_imports_layout = $*,|,* ij_java_indent_case_from_switch = true ij_java_insert_inner_class_imports = false ij_java_insert_override_annotation = true ij_java_keep_blank_lines_before_right_brace = 2 ij_java_keep_blank_lines_between_package_declaration_and_header = 2 ij_java_keep_blank_lines_in_code = 2 ij_java_keep_blank_lines_in_declarations = 2 ij_java_keep_builder_methods_indents = false ij_java_keep_control_statement_in_one_line = true ij_java_keep_first_column_comment = true ij_java_keep_indents_on_empty_lines = false ij_java_keep_line_breaks = true ij_java_keep_multiple_expressions_in_one_line = false ij_java_keep_simple_blocks_in_one_line = false ij_java_keep_simple_classes_in_one_line = false ij_java_keep_simple_lambdas_in_one_line = false ij_java_keep_simple_methods_in_one_line = false ij_java_label_indent_absolute = false ij_java_label_indent_size = 0 ij_java_lambda_brace_style = end_of_line ij_java_layout_static_imports_separately = true ij_java_line_comment_add_space = false ij_java_line_comment_add_space_on_reformat = false ij_java_line_comment_at_first_column = true ij_java_message_dd_suffix = EJB ij_java_message_eb_suffix = Bean ij_java_method_annotation_wrap = split_into_lines ij_java_method_brace_style = end_of_line ij_java_method_call_chain_wrap = normal ij_java_method_parameters_new_line_after_left_paren = false ij_java_method_parameters_right_paren_on_new_line = false ij_java_method_parameters_wrap = normal ij_java_modifier_list_wrap = false ij_java_multi_catch_types_wrap = normal ij_java_names_count_to_use_import_on_demand = 999 ij_java_new_line_after_lparen_in_annotation = false ij_java_new_line_after_lparen_in_record_header = false ij_java_parameter_annotation_wrap = normal ij_java_parentheses_expression_new_line_after_left_paren = false ij_java_parentheses_expression_right_paren_on_new_line = false ij_java_place_assignment_sign_on_next_line = false ij_java_prefer_longer_names = true ij_java_prefer_parameters_wrap = false ij_java_record_components_wrap = normal ij_java_repeat_synchronized = true ij_java_replace_instanceof_and_cast = false ij_java_replace_null_check = true ij_java_replace_sum_lambda_with_method_ref = true ij_java_resource_list_new_line_after_left_paren = false ij_java_resource_list_right_paren_on_new_line = false ij_java_resource_list_wrap = normal ij_java_rparen_on_new_line_in_annotation = false ij_java_rparen_on_new_line_in_record_header = false ij_java_session_dd_suffix = EJB ij_java_session_eb_suffix = Bean ij_java_session_hi_suffix = Home ij_java_session_lhi_prefix = Local ij_java_session_lhi_suffix = Home ij_java_session_li_prefix = Local ij_java_session_si_suffix = Service ij_java_space_after_closing_angle_bracket_in_type_argument = false ij_java_space_after_colon = true ij_java_space_after_comma = true ij_java_space_after_comma_in_type_arguments = true ij_java_space_after_for_semicolon = true ij_java_space_after_quest = true ij_java_space_after_type_cast = true ij_java_space_before_annotation_array_initializer_left_brace = false ij_java_space_before_annotation_parameter_list = false ij_java_space_before_array_initializer_left_brace = true ij_java_space_before_catch_keyword = true ij_java_space_before_catch_left_brace = true ij_java_space_before_catch_parentheses = true ij_java_space_before_class_left_brace = true ij_java_space_before_colon = true ij_java_space_before_colon_in_foreach = true ij_java_space_before_comma = false ij_java_space_before_do_left_brace = true ij_java_space_before_else_keyword = true ij_java_space_before_else_left_brace = true ij_java_space_before_finally_keyword = true ij_java_space_before_finally_left_brace = true ij_java_space_before_for_left_brace = true ij_java_space_before_for_parentheses = true ij_java_space_before_for_semicolon = false ij_java_space_before_if_left_brace = true ij_java_space_before_if_parentheses = true ij_java_space_before_method_call_parentheses = false ij_java_space_before_method_left_brace = true ij_java_space_before_method_parentheses = false ij_java_space_before_opening_angle_bracket_in_type_parameter = false ij_java_space_before_quest = true ij_java_space_before_switch_left_brace = true ij_java_space_before_switch_parentheses = true ij_java_space_before_synchronized_left_brace = true ij_java_space_before_synchronized_parentheses = true ij_java_space_before_try_left_brace = true ij_java_space_before_try_parentheses = true ij_java_space_before_type_parameter_list = false ij_java_space_before_while_keyword = true ij_java_space_before_while_left_brace = true ij_java_space_before_while_parentheses = true ij_java_space_inside_one_line_enum_braces = false ij_java_space_within_empty_array_initializer_braces = false ij_java_space_within_empty_method_call_parentheses = false ij_java_space_within_empty_method_parentheses = false ij_java_spaces_around_additive_operators = true ij_java_spaces_around_annotation_eq = true ij_java_spaces_around_assignment_operators = true ij_java_spaces_around_bitwise_operators = true ij_java_spaces_around_equality_operators = true ij_java_spaces_around_lambda_arrow = true ij_java_spaces_around_logical_operators = true ij_java_spaces_around_method_ref_dbl_colon = false ij_java_spaces_around_multiplicative_operators = true ij_java_spaces_around_relational_operators = true ij_java_spaces_around_shift_operators = true ij_java_spaces_around_type_bounds_in_type_parameters = true ij_java_spaces_around_unary_operator = false ij_java_spaces_within_angle_brackets = false ij_java_spaces_within_annotation_parentheses = false ij_java_spaces_within_array_initializer_braces = false ij_java_spaces_within_braces = false ij_java_spaces_within_brackets = false ij_java_spaces_within_cast_parentheses = false ij_java_spaces_within_catch_parentheses = false ij_java_spaces_within_for_parentheses = false ij_java_spaces_within_if_parentheses = false ij_java_spaces_within_method_call_parentheses = false ij_java_spaces_within_method_parentheses = false ij_java_spaces_within_parentheses = false ij_java_spaces_within_record_header = false ij_java_spaces_within_switch_parentheses = false ij_java_spaces_within_synchronized_parentheses = false ij_java_spaces_within_try_parentheses = false ij_java_spaces_within_while_parentheses = false ij_java_special_else_if_treatment = true ij_java_subclass_name_suffix = Impl ij_java_ternary_operation_signs_on_next_line = false ij_java_ternary_operation_wrap = normal ij_java_test_name_suffix = Test ij_java_throws_keyword_wrap = normal ij_java_throws_list_wrap = normal ij_java_use_external_annotations = false ij_java_use_fq_class_names = false ij_java_use_relative_indents = false ij_java_use_single_class_imports = true ij_java_variable_annotation_wrap = normal ij_java_visibility = public ij_java_while_brace_force = always ij_java_while_on_new_line = false ij_java_wrap_comments = false ij_java_wrap_first_method_in_call_chain = false ij_java_wrap_long_lines = false [*.md] insert_final_newline = false trim_trailing_whitespace = false [*.yaml] indent_size = 2 [*.yml] indent_size = 2 ================================================ FILE: .github/CODEOWNERS ================================================ * @Haarolean # BACKEND /pom.xml @provectus/kafka-backend /kafka-ui-contract/ @provectus/kafka-backend /kafka-ui-api/ @provectus/kafka-backend # FRONTEND /kafka-ui-react-app/ @provectus/kafka-frontend # TESTS /kafka-ui-e2e-checks/ @provectus/kafka-qa # INFRA /.github/workflows/ @provectus/kafka-devops ================================================ FILE: .github/ISSUE_TEMPLATE/bug.yml ================================================ name: "\U0001F41E Bug report" description: File a bug report labels: ["status/triage", "type/bug"] assignees: [] body: - type: markdown attributes: value: | Hi, thanks for raising the issue(-s), all contributions really matter! Please, note that we'll close the issue without further explanation if you don't follow this template and don't provide the information requested within this template. - type: checkboxes id: terms attributes: label: Issue submitter TODO list description: By you checking these checkboxes we can be sure you've done the essential things. options: - label: I've looked up my issue in [FAQ](https://docs.kafka-ui.provectus.io/faq/common-problems) required: true - label: I've searched for an already existing issues [here](https://github.com/provectus/kafka-ui/issues) required: true - label: I've tried running `master`-labeled docker image and the issue still persists there required: true - label: I'm running a supported version of the application which is listed [here](https://github.com/provectus/kafka-ui/blob/master/SECURITY.md) required: true - type: textarea attributes: label: Describe the bug (actual behavior) description: A clear and concise description of what the bug is. Use a list, if there is more than one problem validations: required: true - type: textarea attributes: label: Expected behavior description: A clear and concise description of what you expected to happen validations: required: false - type: textarea attributes: label: Your installation details description: | How do you run the app? Please provide as much info as possible: 1. App version (commit hash in the top left corner of the UI) 2. Helm chart version, if you use one 3. Your application config. Please remove the sensitive info like passwords or API keys. 4. Any IAAC configs validations: required: true - type: textarea attributes: label: Steps to reproduce description: | Please write down the order of the actions required to reproduce the issue. For the advanced setups/complicated issue, we might need you to provide a minimal [reproducible example](https://stackoverflow.com/help/minimal-reproducible-example). validations: required: true - type: textarea attributes: label: Screenshots description: | If applicable, add screenshots to help explain your problem validations: required: false - type: textarea attributes: label: Logs description: | If applicable, *upload* screenshots to help explain your problem validations: required: false - type: textarea attributes: label: Additional context description: | Add any other context about the problem here. E.G.: 1. Are there any alternative scenarios (different data/methods/configuration/setup) you have tried? Were they successful or the same issue occurred? Please provide steps as well. 2. Related issues (if there are any). 3. Logs (if available) 4. Is there any serious impact or behaviour on the end-user because of this issue, that can be overlooked? validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Report helm issue url: https://github.com/provectus/kafka-ui-charts about: Our helm charts are located in another repo. Please raise issues/PRs regarding charts in that repo. - name: Official documentation url: https://docs.kafka-ui.provectus.io/ about: Before reaching out for support, please refer to our documentation. Read "FAQ" and "Common problems", also try using search there. - name: Community Discord url: https://discord.gg/4DWzD7pGE5 about: Chat with other users, get some support or ask questions. - name: GitHub Discussions url: https://github.com/provectus/kafka-ui/discussions about: An alternative place to ask questions or to get some support. ================================================ FILE: .github/ISSUE_TEMPLATE/feature.yml ================================================ name: "\U0001F680 Feature request" description: Propose a new feature labels: ["status/triage", "type/feature"] assignees: [] body: - type: markdown attributes: value: | Hi, thanks for raising the issue(-s), all contributions really matter! Please, note that we'll close the issue without further explanation if you don't follow this template and don't provide the information requested within this template. - type: checkboxes id: terms attributes: label: Issue submitter TODO list description: By you checking these checkboxes we can be sure you've done the essential things. options: - label: I've searched for an already existing issues [here](https://github.com/provectus/kafka-ui/issues) required: true - label: I'm running a supported version of the application which is listed [here](https://github.com/provectus/kafka-ui/blob/master/SECURITY.md) and the feature is not present there required: true - type: textarea attributes: label: Is your proposal related to a problem? description: | Provide a clear and concise description of what the problem is. For example, "I'm always frustrated when..." validations: required: false - type: textarea attributes: label: Describe the feature you're interested in description: | Provide a clear and concise description of what you want to happen. validations: required: true - type: textarea attributes: label: Describe alternatives you've considered description: | Let us know about other solutions you've tried or researched. validations: required: false - type: input attributes: label: Version you're running description: | Please provide the app version you're currently running: 1. App version (commit hash in the top left corner of the UI) validations: required: true - type: textarea attributes: label: Additional context description: | Is there anything else you can add about the proposal? You might want to link to related issues here, if you haven't already. validations: required: false ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ - [ ] **Breaking change?** (if so, please describe the impact and migration path for existing application instances) **What changes did you make?** (Give an overview) **Is there anything you'd like reviewers to focus on?** **How Has This Been Tested?** (put an "x" (case-sensitive!) next to an item) - [ ] No need to - [ ] Manually (please, describe, if necessary) - [ ] Unit checks - [ ] Integration checks - [ ] Covered by existing automation **Checklist** (put an "x" (case-sensitive!) next to all the items, otherwise the build will fail) - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation (e.g. **ENVIRONMENT VARIABLES**) - [ ] My changes generate no new warnings (e.g. Sonar is happy) - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged Check out [Contributing](https://github.com/provectus/kafka-ui/blob/master/CONTRIBUTING.md) and [Code of Conduct](https://github.com/provectus/kafka-ui/blob/master/CODE-OF-CONDUCT.md) **A picture of a cute animal (not mandatory but encouraged)** ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: maven directory: "/" schedule: interval: daily time: "10:00" timezone: Europe/Moscow reviewers: - "Haarolean" labels: - "scope/backend" - "type/dependencies" - package-ecosystem: npm directory: "/kafka-ui-react-app" schedule: interval: weekly time: "10:00" timezone: Europe/Moscow open-pull-requests-limit: 10 versioning-strategy: increase-if-necessary labels: - "scope/frontend" - "type/dependencies" ignore: - dependency-name: react-hook-form versions: - 6.15.5 - 7.0.0 - 7.0.6 - dependency-name: "@hookform/error-message" versions: - 1.1.0 - dependency-name: use-debounce versions: - 6.0.0 - 6.0.1 - dependency-name: "@rooks/use-outside-click-ref" versions: - 4.10.1 - dependency-name: react-multi-select-component versions: - 3.1.6 - 4.0.0 - dependency-name: husky versions: - 5.1.3 - 5.2.0 - 6.0.0 - dependency-name: "@types/node-fetch" versions: - 2.5.9 - dependency-name: "@testing-library/jest-dom" versions: - 5.11.10 - dependency-name: "@typescript-eslint/eslint-plugin" versions: - 4.20.0 - dependency-name: "@openapitools/openapi-generator-cli" versions: - 2.2.5 - dependency-name: "@typescript-eslint/parser" versions: - 4.20.0 - dependency-name: react-datepicker versions: - 3.7.0 - dependency-name: eslint versions: - 7.23.0 - dependency-name: "@testing-library/user-event" versions: - 13.0.6 - dependency-name: immer versions: - 9.0.1 - dependency-name: react-scripts versions: - 4.0.3 - dependency-name: eslint-config-prettier versions: - 8.1.0 - dependency-name: "@testing-library/react" versions: - 11.2.5 - dependency-name: lodash versions: - 4.17.21 - dependency-name: react-json-tree versions: - 0.15.0 - package-ecosystem: "github-actions" directory: "/" schedule: interval: weekly time: "10:00" timezone: Europe/Moscow reviewers: - "Haarolean" labels: - "scope/infrastructure" - "type/dependencies" ================================================ FILE: .github/release_drafter.yaml ================================================ name-template: '$RESOLVED_VERSION' tag-template: 'v$RESOLVED_VERSION' template: | ## Changes $CHANGES ## Contributors $CONTRIBUTORS exclude-labels: - 'scope/infrastructure' - 'scope/QA' - 'scope/AQA' - 'type/dependencies' - 'type/chore' - 'type/documentation' - 'type/refactoring' categories: - title: '🚩 Breaking Changes' labels: - 'impact/changelog' - title: '⚙️Features' labels: - 'type/feature' - title: '🪛Enhancements' labels: - 'type/enhancement' - title: '🔨Bug Fixes' labels: - 'type/bug' - title: 'Security' labels: - 'type/security' - title: '⎈ Helm/K8S Changes' labels: - 'scope/k8s' change-template: '- $TITLE @$AUTHOR (#$NUMBER)' version-resolver: major: labels: - 'major' minor: labels: - 'minor' patch: labels: - 'patch' default: patch ================================================ FILE: .github/workflows/aws_publisher.yaml ================================================ name: "Infra: Release: AWS Marketplace Publisher" on: workflow_dispatch: inputs: KafkaUIInfraBranch: description: 'Branch name of Kafka-UI-Infra repo, build commands will be executed from this branch' required: true default: 'master' KafkaUIReleaseVersion: description: 'Version of KafkaUI' required: true default: '0.3.2' PublishOnMarketplace: description: 'If set to true, the request to update AWS Server product version will be raised' required: true default: false type: boolean jobs: build-ami: name: Build AMI runs-on: ubuntu-latest steps: - name: Clone infra repo run: | echo "Cloning repo..." git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch ${{ github.event.inputs.KafkaUIInfraBranch }} echo "Cd to packer DIR..." cd kafka-ui-infra/ami echo "WORK_DIR=$(pwd)" >> $GITHUB_ENV echo "Packer will be triggered in this dir $WORK_DIR" - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_AMI_PUBLISH_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_AMI_PUBLISH_KEY_SECRET }} aws-region: us-east-1 # validate templates - name: Validate Template uses: hashicorp/packer-github-actions@master with: command: validate arguments: -syntax-only target: kafka-ui-infra/ami/kafka-ui.pkr.hcl # build artifact - name: Build Artifact uses: hashicorp/packer-github-actions@master with: command: build arguments: "-color=false -on-error=abort -var=kafka_ui_release_version=${{ github.event.inputs.KafkaUIReleaseVersion }}" target: kafka-ui.pkr.hcl working_directory: ${{ env.WORK_DIR }} env: PACKER_LOG: 1 # add fresh AMI to AWS Marketplace - name: Publish Artifact at Marketplace if: ${{ github.event.inputs.PublishOnMarketplace == 'true' }} env: PRODUCT_ID: ${{ secrets.AWS_SERVER_PRODUCT_ID }} RELEASE_VERSION: "${{ github.event.inputs.KafkaUIReleaseVersion }}" RELEASE_NOTES: "https://github.com/provectus/kafka-ui/releases/tag/v${{ github.event.inputs.KafkaUIReleaseVersion }}" MP_ROLE_ARN: ${{ secrets.AWS_MARKETPLACE_AMI_ACCESS_ROLE }} # https://docs.aws.amazon.com/marketplace/latest/userguide/ami-single-ami-products.html#single-ami-marketplace-ami-access AMI_OS_VERSION: "amzn2-ami-kernel-5.10-hvm-*-x86_64-gp2" run: | set -x pwd ls -la kafka-ui-infra/ami echo $WORK_DIR/manifest.json export AMI_ID=$(jq -r '.builds[-1].artifact_id' kafka-ui-infra/ami/manifest.json | cut -d ":" -f2) /bin/bash kafka-ui-infra/aws-marketplace/prepare_changeset.sh > changeset.json aws marketplace-catalog start-change-set \ --catalog "AWSMarketplace" \ --change-set "$(cat changeset.json)" ================================================ FILE: .github/workflows/backend.yml ================================================ name: "Backend: PR/master build & test" on: push: branches: - master pull_request_target: types: ["opened", "edited", "reopened", "synchronize"] paths: - "kafka-ui-api/**" - "pom.xml" permissions: checks: write pull-requests: write jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Cache SonarCloud packages uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Build and analyze pull request target if: ${{ github.event_name == 'pull_request' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_BACKEND }} HEAD_REF: ${{ github.head_ref }} BASE_REF: ${{ github.base_ref }} run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.event.pull_request.head.sha }} ./mvnw -B -V -ntp verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dsonar.projectKey=com.provectus:kafka-ui_backend \ -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} \ -Dsonar.pullrequest.branch=$HEAD_REF \ -Dsonar.pullrequest.base=$BASE_REF - name: Build and analyze push master if: ${{ github.event_name == 'push' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_BACKEND }} run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -B -V -ntp verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dsonar.projectKey=com.provectus:kafka-ui_backend ================================================ FILE: .github/workflows/block_merge.yml ================================================ name: "Infra: PR block merge" on: pull_request: types: [opened, labeled, unlabeled, synchronize] jobs: block_merge: runs-on: ubuntu-latest steps: - uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 labels: "status/blocked, status/needs-attention, status/on-hold, status/pending, status/triage, status/pending-backend, status/pending-frontend, status/pending-QA" ================================================ FILE: .github/workflows/branch-deploy.yml ================================================ name: "Infra: Feature Testing: Init env" on: workflow_dispatch: pull_request: types: ['labeled'] jobs: build: if: ${{ github.event.label.name == 'status/feature_testing' || github.event.label.name == 'status/feature_testing_public' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: get branch name id: extract_branch run: | tag='pr${{ github.event.pull_request.number }}' echo "tag=${tag}" >> $GITHUB_OUTPUT env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -B -V -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - name: Build and push id: docker_build_and_push uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api push: true tags: 297478128798.dkr.ecr.eu-central-1.amazonaws.com/kafka-ui:${{ steps.extract_branch.outputs.tag }} build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache outputs: tag: ${{ steps.extract_branch.outputs.tag }} make-branch-env: needs: build runs-on: ubuntu-latest steps: - name: clone run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch envs - name: create deployment run: | cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts echo "Branch:${{ needs.build.outputs.tag }}" ./kafka-ui-deployment-from-branch.sh ${{ needs.build.outputs.tag }} ${{ github.event.label.name }} ${{ secrets.FEATURE_TESTING_UI_PASSWORD }} git config --global user.email "infra-tech@provectus.com" git config --global user.name "infra-tech" git add ../kafka-ui-from-branch/ git commit -m "added env:${{ needs.build.outputs.deploy }}" && git push || true - name: update status check for private deployment if: ${{ github.event.label.name == 'status/feature_testing' }} uses: Sibz/github-status-action@v1.1.6 with: authToken: ${{secrets.GITHUB_TOKEN}} context: "Click Details button to open custom deployment page" state: "success" sha: ${{ github.event.pull_request.head.sha || github.sha }} target_url: "http://${{ needs.build.outputs.tag }}.internal.kafka-ui.provectus.io" - name: update status check for public deployment if: ${{ github.event.label.name == 'status/feature_testing_public' }} uses: Sibz/github-status-action@v1.1.6 with: authToken: ${{secrets.GITHUB_TOKEN}} context: "Click Details button to open custom deployment page" state: "success" sha: ${{ github.event.pull_request.head.sha || github.sha }} target_url: "http://${{ needs.build.outputs.tag }}.internal.kafka-ui.provectus.io" ================================================ FILE: .github/workflows/branch-remove.yml ================================================ name: "Infra: Feature Testing: Destroy env" on: workflow_dispatch: pull_request: types: ['unlabeled', 'closed'] jobs: remove: runs-on: ubuntu-latest if: ${{ (github.event.label.name == 'status/feature_testing' || github.event.label.name == 'status/feature_testing_public') || (github.event.action == 'closed' && (contains(github.event.pull_request.labels.*.name, 'status/feature_testing') || contains(github.event.pull_request.labels.*.name, 'status/feature_testing_public'))) }} steps: - uses: actions/checkout@v3 - name: clone run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch envs - name: remove env run: | cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts ./delete-env.sh pr${{ github.event.pull_request.number }} || true git config --global user.email "infra-tech@provectus.com" git config --global user.name "infra-tech" git add ../kafka-ui-from-branch/ git commit -m "removed env:${{ needs.build.outputs.deploy }}" && git push || true ================================================ FILE: .github/workflows/build-public-image.yml ================================================ name: "Infra: Image Testing: Deploy" on: workflow_dispatch: pull_request: types: ['labeled'] jobs: build: if: ${{ github.event.label.name == 'status/image_testing' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: get branch name id: extract_branch run: | tag='${{ github.event.pull_request.number }}' echo "tag=${tag}" >> $GITHUB_OUTPUT - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -B -V -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 with: registry-type: 'public' - name: Build and push id: docker_build_and_push uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api push: true tags: public.ecr.aws/provectus/kafka-ui-custom-build:${{ steps.extract_branch.outputs.tag }} build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: make comment with private deployment link uses: peter-evans/create-or-update-comment@v3 with: issue-number: ${{ github.event.pull_request.number }} body: | Image published at public.ecr.aws/provectus/kafka-ui-custom-build:${{ steps.extract_branch.outputs.tag }} outputs: tag: ${{ steps.extract_branch.outputs.tag }} ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] paths: - 'kafka-ui-contract/**' - 'kafka-ui-react-app/**' - 'kafka-ui-api/**' - 'kafka-ui-serde-api/**' schedule: - cron: '39 15 * * 6' jobs: analyze: name: Analyze runs-on: ubuntu-latest strategy: fail-fast: false matrix: language: [ 'javascript', 'java' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 ================================================ FILE: .github/workflows/cve.yaml ================================================ name: CVE checks docker master on: workflow_dispatch: schedule: # * is a special character in YAML so you have to quote this string - cron: '0 8 15 * *' jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build project id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -B -V -ntp clean package -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Build docker image uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api platforms: linux/amd64 push: false load: true tags: | provectuslabs/kafka-ui:${{ steps.build.outputs.version }} build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Run CVE checks uses: aquasecurity/trivy-action@0.12.0 with: image-ref: "provectuslabs/kafka-ui:${{ steps.build.outputs.version }}" format: "table" exit-code: "1" ================================================ FILE: .github/workflows/delete-public-image.yml ================================================ name: "Infra: Image Testing: Delete" on: workflow_dispatch: pull_request: types: ['unlabeled', 'closed'] jobs: remove: if: ${{ github.event.label.name == 'status/image_testing' || ( github.event.action == 'closed' && (contains(github.event.pull_request.labels, 'status/image_testing'))) }} runs-on: ubuntu-latest steps: - name: get branch name id: extract_branch run: | echo tag='${{ github.event.pull_request.number }}' echo "tag=${tag}" >> $GITHUB_OUTPUT - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 with: registry-type: 'public' - name: Remove from ECR id: remove_from_ecr run: | aws ecr-public batch-delete-image \ --repository-name kafka-ui-custom-build \ --image-ids imageTag=${{ steps.extract_branch.outputs.tag }} \ --region us-east-1 ================================================ FILE: .github/workflows/documentation.yaml ================================================ name: "Infra: Docs: URL linter" on: pull_request: types: - opened - labeled - reopened - synchronize paths: - 'documentation/**' - '**.md' jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Check URLs in files uses: urlstechie/urlchecker-action@0.0.34 with: exclude_patterns: localhost,127.0.,192.168. exclude_urls: https://api.server,https://graph.microsoft.com/User.Read,https://dev-a63ggcut.auth0.com/,http://main-schema-registry:8081,http://schema-registry:8081,http://another-yet-schema-registry:8081,http://another-schema-registry:8081 print_all: false file_types: .md ================================================ FILE: .github/workflows/e2e-automation.yml ================================================ name: "E2E: Automation suite" on: workflow_dispatch: inputs: test_suite: description: 'Select test suite to run' default: 'regression' required: true type: choice options: - regression - sanity - smoke qase_token: description: 'Set Qase token to enable integration' required: false type: string jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.sha }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Set up environment id: set_env_values run: | cat "./kafka-ui-e2e-checks/.env.ci" >> "./kafka-ui-e2e-checks/.env" - name: Pull with Docker id: pull_chrome run: | docker pull selenoid/vnc_chrome:103.0 - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build with Maven id: build_app run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp clean install -Pprod -Dmaven.test.skip=true ${{ github.event.inputs.extraMavenOptions }} - name: Compose with Docker id: compose_app # use the following command until #819 will be fixed run: | docker-compose -f kafka-ui-e2e-checks/docker/selenoid-git.yaml up -d docker-compose -f ./documentation/compose/e2e-tests.yaml up -d - name: Run test suite run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp -DQASEIO_API_TOKEN=${{ github.event.inputs.qase_token }} -Dsurefire.suiteXmlFiles='src/test/resources/${{ github.event.inputs.test_suite }}.xml' -Dsuite=${{ github.event.inputs.test_suite }} -f 'kafka-ui-e2e-checks' test -Pprod - name: Generate Allure report uses: simple-elf/allure-report-action@master if: always() id: allure-report with: allure_results: ./kafka-ui-e2e-checks/allure-results gh_pages: allure-results allure_report: allure-report subfolder: allure-results report_url: "http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com" - uses: jakejarvis/s3-sync-action@master if: always() env: AWS_S3_BUCKET: 'kafkaui-allure-reports' AWS_REGION: 'eu-central-1' SOURCE_DIR: 'allure-history/allure-results' - name: Deploy report to Amazon S3 if: always() uses: Sibz/github-status-action@v1.1.6 with: authToken: ${{secrets.GITHUB_TOKEN}} context: "Click Details button to open Allure report" state: "success" sha: ${{ github.sha }} target_url: http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com/${{ github.run_number }} - name: Dump Docker logs on failure if: failure() uses: jwalton/gh-docker-logs@v2.2.1 ================================================ FILE: .github/workflows/e2e-checks.yaml ================================================ name: "E2E: PR healthcheck" on: pull_request_target: types: [ "opened", "edited", "reopened", "synchronize" ] paths: - "kafka-ui-api/**" - "kafka-ui-contract/**" - "kafka-ui-react-app/**" - "kafka-ui-e2e-checks/**" - "pom.xml" permissions: statuses: write jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.S3_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Set up environment id: set_env_values run: | cat "./kafka-ui-e2e-checks/.env.ci" >> "./kafka-ui-e2e-checks/.env" - name: Pull with Docker id: pull_chrome run: | docker pull selenoid/vnc_chrome:103.0 - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build with Maven id: build_app run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.event.pull_request.head.sha }} ./mvnw -B -V -ntp clean install -Pprod -Dmaven.test.skip=true ${{ github.event.inputs.extraMavenOptions }} - name: Compose with Docker id: compose_app # use the following command until #819 will be fixed run: | docker-compose -f kafka-ui-e2e-checks/docker/selenoid-git.yaml up -d docker-compose -f ./documentation/compose/e2e-tests.yaml up -d && until [ "$(docker exec kafka-ui wget --spider --server-response http://localhost:8080/actuator/health 2>&1 | grep -c 'HTTP/1.1 200 OK')" == "1" ]; do echo "Waiting for kafka-ui ..." && sleep 1; done - name: Run test suite run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.event.pull_request.head.sha }} ./mvnw -B -V -ntp -Dsurefire.suiteXmlFiles='src/test/resources/smoke.xml' -f 'kafka-ui-e2e-checks' test -Pprod - name: Generate allure report uses: simple-elf/allure-report-action@master if: always() id: allure-report with: allure_results: ./kafka-ui-e2e-checks/allure-results gh_pages: allure-results allure_report: allure-report subfolder: allure-results report_url: "http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com" - uses: jakejarvis/s3-sync-action@master if: always() env: AWS_S3_BUCKET: 'kafkaui-allure-reports' AWS_REGION: 'eu-central-1' SOURCE_DIR: 'allure-history/allure-results' - name: Deploy report to Amazon S3 if: always() uses: Sibz/github-status-action@v1.1.6 with: authToken: ${{secrets.GITHUB_TOKEN}} context: "Click Details button to open Allure report" state: "success" sha: ${{ github.event.pull_request.head.sha || github.sha }} target_url: http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com/${{ github.run_number }} - name: Dump docker logs on failure if: failure() uses: jwalton/gh-docker-logs@v2.2.1 ================================================ FILE: .github/workflows/e2e-manual.yml ================================================ name: "E2E: Manual suite" on: workflow_dispatch: inputs: test_suite: description: 'Select test suite to run' default: 'manual' required: true type: choice options: - manual - qase qase_token: description: 'Set Qase token to enable integration' required: true type: string jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.sha }} - name: Set up environment id: set_env_values run: | cat "./kafka-ui-e2e-checks/.env.ci" >> "./kafka-ui-e2e-checks/.env" - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build with Maven id: build_app run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp clean install -Pprod -Dmaven.test.skip=true ${{ github.event.inputs.extraMavenOptions }} - name: Run test suite run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp -DQASEIO_API_TOKEN=${{ github.event.inputs.qase_token }} -Dsurefire.suiteXmlFiles='src/test/resources/${{ github.event.inputs.test_suite }}.xml' -Dsuite=${{ github.event.inputs.test_suite }} -f 'kafka-ui-e2e-checks' test -Pprod ================================================ FILE: .github/workflows/e2e-weekly.yml ================================================ name: "E2E: Weekly suite" on: schedule: - cron: '0 1 * * 1' jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.sha }} - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Set up environment id: set_env_values run: | cat "./kafka-ui-e2e-checks/.env.ci" >> "./kafka-ui-e2e-checks/.env" - name: Pull with Docker id: pull_chrome run: | docker pull selenoid/vnc_chrome:103.0 - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build with Maven id: build_app run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp clean install -Pprod -Dmaven.test.skip=true ${{ github.event.inputs.extraMavenOptions }} - name: Compose with Docker id: compose_app # use the following command until #819 will be fixed run: | docker-compose -f kafka-ui-e2e-checks/docker/selenoid-git.yaml up -d docker-compose -f ./documentation/compose/e2e-tests.yaml up -d - name: Run test suite run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.sha }} ./mvnw -B -V -ntp -DQASEIO_API_TOKEN=${{ secrets.QASEIO_API_TOKEN }} -Dsurefire.suiteXmlFiles='src/test/resources/sanity.xml' -Dsuite=weekly -f 'kafka-ui-e2e-checks' test -Pprod - name: Generate Allure report uses: simple-elf/allure-report-action@master if: always() id: allure-report with: allure_results: ./kafka-ui-e2e-checks/allure-results gh_pages: allure-results allure_report: allure-report subfolder: allure-results report_url: "http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com" - uses: jakejarvis/s3-sync-action@master if: always() env: AWS_S3_BUCKET: 'kafkaui-allure-reports' AWS_REGION: 'eu-central-1' SOURCE_DIR: 'allure-history/allure-results' - name: Deploy report to Amazon S3 if: always() uses: Sibz/github-status-action@v1.1.6 with: authToken: ${{secrets.GITHUB_TOKEN}} context: "Click Details button to open Allure report" state: "success" sha: ${{ github.sha }} target_url: http://kafkaui-allure-reports.s3-website.eu-central-1.amazonaws.com/${{ github.run_number }} - name: Dump Docker logs on failure if: failure() uses: jwalton/gh-docker-logs@v2.2.1 ================================================ FILE: .github/workflows/frontend.yaml ================================================ name: "Frontend: PR/master build & test" on: push: branches: - master pull_request_target: types: ["opened", "edited", "reopened", "synchronize"] paths: - "kafka-ui-contract/**" - "kafka-ui-react-app/**" permissions: checks: write pull-requests: write jobs: build-and-test: env: CI: true NODE_ENV: dev runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: # Disabling shallow clone is recommended for improving relevancy of reporting fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - uses: pnpm/action-setup@v2.4.0 with: version: 8.6.12 - name: Install node uses: actions/setup-node@v3.8.1 with: node-version: "18.17.1" cache: "pnpm" cache-dependency-path: "./kafka-ui-react-app/pnpm-lock.yaml" - name: Install Node dependencies run: | cd kafka-ui-react-app/ pnpm install --frozen-lockfile - name: Generate sources run: | cd kafka-ui-react-app/ pnpm gen:sources - name: Linter run: | cd kafka-ui-react-app/ pnpm lint:CI - name: Tests run: | cd kafka-ui-react-app/ pnpm test:CI - name: SonarCloud Scan uses: sonarsource/sonarcloud-github-action@master with: projectBaseDir: ./kafka-ui-react-app args: -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} -Dsonar.pullrequest.branch=${{ github.head_ref }} -Dsonar.pullrequest.base=${{ github.base_ref }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN_FRONTEND }} ================================================ FILE: .github/workflows/master.yaml ================================================ name: "Master: Build & deploy" on: workflow_dispatch: push: branches: [ "master" ] jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -V -B -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT ################# # # # Docker images # # # ################# - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push id: docker_build_and_push uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api platforms: linux/amd64,linux/arm64 provenance: false push: true tags: | provectuslabs/kafka-ui:${{ steps.build.outputs.version }} provectuslabs/kafka-ui:master build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache ################################# # # # Master image digest update # # # ################################# - name: update-master-deployment run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch master cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts echo "Image digest is:${{ steps.docker_build_and_push.outputs.digest }}" ./kafka-ui-update-master-digest.sh ${{ steps.docker_build_and_push.outputs.digest }} git config --global user.email "infra-tech@provectus.com" git config --global user.name "infra-tech" git add ../kafka-ui/* git commit -m "updated master image digest: ${{ steps.docker_build_and_push.outputs.digest }}" && git push ================================================ FILE: .github/workflows/pr-checks.yaml ================================================ name: "PR: Checklist linter" on: pull_request_target: types: [opened, edited, synchronize, reopened] permissions: checks: write jobs: task-check: runs-on: ubuntu-latest steps: - uses: kentaro-m/task-completed-checker-action@v0.1.2 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" - uses: dekinderfiets/pr-description-enforcer@0.0.1 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" ================================================ FILE: .github/workflows/release-serde-api.yaml ================================================ name: "Infra: Release: Serde API" on: workflow_dispatch jobs: release-serde-api: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - run: | git config user.name github-actions git config user.email github-actions@github.com - name: Set up JDK uses: actions/setup-java@v3 with: java-version: "17" distribution: "zulu" cache: "maven" - id: install-secret-key name: Install GPG secret key run: | cat <(echo -e "${{ secrets.GPG_PRIVATE_KEY }}") | gpg --batch --import - name: Publish to Maven Central run: | mvn source:jar javadoc:jar package gpg:sign -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} -Dserver.username=${{ secrets.NEXUS_USERNAME }} -Dserver.password=${{ secrets.NEXUS_PASSWORD }} nexus-staging:deploy -pl kafka-ui-serde-api -s settings.xml ================================================ FILE: .github/workflows/release.yaml ================================================ name: "Infra: Release" on: release: types: [published] jobs: release: runs-on: ubuntu-latest outputs: version: ${{steps.build.outputs.version}} steps: - uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - run: | git config user.name github-actions git config user.email github-actions@github.com - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build with Maven id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=${{ github.event.release.tag_name }} ./mvnw -B -V -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Upload files to a GitHub release uses: svenstaro/upload-release-action@2.7.0 with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: kafka-ui-api/target/kafka-ui-api-${{ steps.build.outputs.version }}.jar tag: ${{ github.event.release.tag_name }} - name: Archive JAR uses: actions/upload-artifact@v3 with: name: kafka-ui-${{ steps.build.outputs.version }} path: kafka-ui-api/target/kafka-ui-api-${{ steps.build.outputs.version }}.jar ################# # # # Docker images # # # ################# - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push id: docker_build_and_push uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api platforms: linux/amd64,linux/arm64 provenance: false push: true tags: | provectuslabs/kafka-ui:${{ steps.build.outputs.version }} provectuslabs/kafka-ui:latest build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache charts: runs-on: ubuntu-latest needs: release steps: - name: Repository Dispatch uses: peter-evans/repository-dispatch@v2 with: token: ${{ secrets.CHARTS_ACTIONS_TOKEN }} repository: provectus/kafka-ui-charts event-type: prepare-helm-release client-payload: '{"appversion": "${{ needs.release.outputs.version }}"}' ================================================ FILE: .github/workflows/release_drafter.yml ================================================ name: "Infra: Release Drafter run" on: push: branches: - master workflow_dispatch: inputs: version: description: 'Release version' required: false branch: description: 'Target branch' required: false default: 'master' permissions: contents: read jobs: update_release_draft: runs-on: ubuntu-latest permissions: contents: write pull-requests: write steps: - uses: release-drafter/release-drafter@v5 with: config-name: release_drafter.yaml disable-autolabeler: true version: ${{ github.event.inputs.version }} commitish: ${{ github.event.inputs.branch }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/separate_env_public_create.yml ================================================ name: "Infra: Feature Testing Public: Init env" on: workflow_dispatch: inputs: ENV_NAME: description: 'Will be used as subdomain in the public URL.' required: true default: 'demo' jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: get branch name id: extract_branch run: | tag="${{ github.event.inputs.ENV_NAME }}-$(date '+%F-%H-%M-%S')" echo "tag=${tag}" >> $GITHUB_OUTPUT env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set up JDK uses: actions/setup-java@v3 with: java-version: '17' distribution: 'zulu' cache: 'maven' - name: Build id: build run: | ./mvnw -B -ntp versions:set -DnewVersion=$GITHUB_SHA ./mvnw -B -V -ntp clean package -Pprod -DskipTests export VERSION=$(./mvnw -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) echo "version=${VERSION}" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - name: Build and push id: docker_build_and_push uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} context: kafka-ui-api push: true tags: 297478128798.dkr.ecr.eu-central-1.amazonaws.com/kafka-ui:${{ steps.extract_branch.outputs.tag }} build-args: | JAR_FILE=kafka-ui-api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache outputs: tag: ${{ steps.extract_branch.outputs.tag }} separate-env-create: runs-on: ubuntu-latest needs: build steps: - name: clone run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch envs - name: separate env create run: | cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts bash separate_env_create.sh ${{ github.event.inputs.ENV_NAME }} ${{ secrets.FEATURE_TESTING_UI_PASSWORD }} ${{ needs.build.outputs.tag }} git config --global user.email "infra-tech@provectus.com" git config --global user.name "infra-tech" git add -A git commit -m "separate env added: ${{ github.event.inputs.ENV_NAME }}" && git push || true - name: echo separate environment public link run: | echo "Please note, separate environment creation takes up to 5-10 minutes." echo "Separate environment will be available at http://${{ github.event.inputs.ENV_NAME }}.kafka-ui.provectus.io" echo "Username: admin" ================================================ FILE: .github/workflows/separate_env_public_remove.yml ================================================ name: "Infra: Feature Testing Public: Destroy env" on: workflow_dispatch: inputs: ENV_NAME: description: 'Will be used to remove previously deployed separate environment.' required: true default: 'demo' jobs: separate-env-remove: runs-on: ubuntu-latest steps: - name: clone run: | git clone https://infra-tech:${{ secrets.INFRA_USER_ACCESS_TOKEN }}@github.com/provectus/kafka-ui-infra.git --branch envs - name: separate environment remove run: | cd kafka-ui-infra/aws-infrastructure4eks/argocd/scripts bash separate_env_remove.sh ${{ github.event.inputs.ENV_NAME }} git config --global user.email "infra-tech@provectus.com" git config --global user.name "infra-tech" git add -A git commit -m "separate env removed: ${{ github.event.inputs.ENV_NAME }}" && git push || true ================================================ FILE: .github/workflows/stale.yaml ================================================ name: 'Infra: Close stale issues' on: schedule: - cron: '30 1 * * *' jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v8 with: days-before-issue-stale: 7 days-before-issue-close: 3 days-before-pr-stale: 7 days-before-pr-close: 7 stale-issue-message: 'This issue has been automatically marked as stale because no requested feedback has been provided. It will be closed if no further activity occurs. Thank you for your contributions.' stale-pr-message: 'This PR has been automatically marked as stale because no requested changes have been applied. It will be closed if no further activity occurs. Thank you for your contributions.' stale-issue-label: 'status/stale' stale-pr-label: 'status/stale' only-labels: 'status/pending' remove-issue-stale-when-updated: true labels-to-remove-when-unstale: 'status/pending' ================================================ FILE: .github/workflows/terraform-deploy.yml ================================================ name: "Infra: Terraform deploy" on: workflow_dispatch: inputs: applyTerraform: description: 'Do you want to apply the infra-repo terraform? Possible values [plan/apply].' required: true default: 'plan' KafkaUIInfraBranch: description: 'Branch name of Kafka-UI-Infra repo, tf will be executed from this branch' required: true default: 'master' jobs: terraform: name: Terraform runs-on: ubuntu-latest steps: - name: Clone infra repo run: | echo "Cloning repo..." git clone https://kafka-ui-infra:${{ secrets.KAFKA_UI_INFRA_TOKEN }}@gitlab.provectus.com/provectus-internals/kafka-ui-infra.git --branch ${{ github.event.inputs.KafkaUIInfraBranch }} echo "Cd to deployment..." cd kafka-ui-infra/aws-infrastructure4eks/deployment echo "TF_DIR=$(pwd)" >> $GITHUB_ENV echo "Terraform will be triggered in this dir $TF_DIR" - name: Configure AWS credentials for Kafka-UI account uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-central-1 - name: Terraform Install uses: hashicorp/setup-terraform@v2 - name: Terraform init id: init run: cd $TF_DIR && terraform init --backend-config="../envs/pro/terraform-backend.tfvars" - name: Terraform validate id: validate run: cd $TF_DIR && terraform validate -no-color - name: Terraform plan id: plan run: | cd $TF_DIR export TF_VAR_github_connector_access_token=${{ secrets.SOURCE_CONNECTOR_GITHUB_TOKEN }} export TF_VAR_repo_secret=${{ secrets.KAFKA_UI_INFRA_TOKEN }} terraform plan --var-file="../envs/pro/eks.tfvars" - name: Terraform apply id: apply if: ${{ github.event.inputs.applyTerraform == 'apply' }} run: | cd $TF_DIR export TF_VAR_github_connector_access_token=${{ secrets.SOURCE_CONNECTOR_GITHUB_TOKEN }} export TF_VAR_repo_secret=${{ secrets.KAFKA_UI_INFRA_TOKEN }} terraform apply --var-file="../envs/pro/eks.tfvars" -auto-approve ================================================ FILE: .github/workflows/triage_issues.yml ================================================ name: "Infra: Triage: Apply triage label for issues" on: issues: types: - opened jobs: triage_issues: runs-on: ubuntu-latest steps: - name: Label issue uses: andymckay/labeler@master with: add-labels: "status/triage" ignore-if-assigned: true ================================================ FILE: .github/workflows/triage_prs.yml ================================================ name: "Infra: Triage: Apply triage label for PRs" on: pull_request: types: - opened jobs: triage_prs: runs-on: ubuntu-latest steps: - name: Label PR uses: andymckay/labeler@master with: add-labels: "status/triage" ignore-if-labeled: true ================================================ FILE: .github/workflows/welcome-first-time-contributors.yml ================================================ name: Welcome first time contributors on: pull_request_target: types: - opened issues: types: - opened permissions: issues: write pull-requests: write jobs: welcome: runs-on: ubuntu-latest steps: - uses: actions/first-interaction@v1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | Hello there ${{ github.actor }}! 👋 Thank you and congratulations 🎉 for opening your very first issue in this project! 💖 In case you want to claim this issue, please comment down below! We will try to get back to you as soon as we can. 👀 pr-message: | Hello there ${{ github.actor }}! 👋 Thank you and congrats 🎉 for opening your first PR on this project! ✨ 💖 We will try to review it soon! ================================================ FILE: .github/workflows/workflow_linter.yaml ================================================ name: "Infra: Workflow linter" on: pull_request: types: - "opened" - "reopened" - "synchronize" - "edited" paths: - ".github/workflows/**" jobs: build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Install yamllint run: sudo apt install -y yamllint - name: Validate workflow yaml files run: yamllint .github/workflows/. -d relaxed -f github --no-warnings ================================================ FILE: .gitignore ================================================ HELP.md target/ !.mvn/wrapper/maven-wrapper.jar !**/src/main/** !**/src/test/** ### STS ### .apt_generated .classpath .factorypath .project .settings .springBeans .sts4-cache ### IntelliJ IDEA ### .idea *.iws *.iml *.ipr ### NetBeans ### /nbproject/private/ /nbbuild/ /dist/ /nbdist/ /.nb-gradle/ build/ ### VS Code ### .vscode/ /kafka-ui-api/app/node ### SDKMAN ### .sdkmanrc .DS_Store *.code-workspace *.tar.gz *.tgz /docker/*.override.yaml ================================================ FILE: .mvn/wrapper/maven-wrapper.properties ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar ================================================ FILE: CODE-OF-CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at email kafkaui@provectus.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. [homepage]: https://www.contributor-covenant.org [v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html [Mozilla CoC]: https://github.com/mozilla/diversity [FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations ================================================ FILE: CONTRIBUTING.md ================================================ This guide is an exact copy of the same documented located [in our official docs](https://docs.kafka-ui.provectus.io/development/contributing). If there are any differences between the documents, the one located in our official docs should prevail. This guide aims to walk you through the process of working on issues and Pull Requests (PRs). Bear in mind that you will not be able to complete some steps on your own if you do not have a “write” permission. Feel free to reach out to the maintainers to help you unlock these activities. # General recommendations Please note that we have a code of conduct (`CODE-OF-CONDUCT.md`). Make sure that you follow it in all of your interactions with the project. # Issues ## Choosing an issue There are two options to look for the issues to contribute to.
The first is our ["Up for grabs"](https://github.com/provectus/kafka-ui/projects/11) board. There the issues are sorted by a required experience level (beginner, intermediate, expert). The second option is to search for ["good first issue"](https://github.com/provectus/kafka-ui/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)-labeled issues. Some of them might not be displayed on the aforementioned board, or vice versa. You also need to consider labels. You can sort the issues by scope labels, such as `scope/backend`, `scope/frontend` or even `scope/k8s`. If any issue covers several specific areas, and you do not have a required expertise for one of them, just do your part of work — others will do the rest. ## Grabbing the issue There is a bunch of criteria that make an issue feasible for development.
The implementation of any features and/or their enhancements should be reasonable, must be backed by justified requirements (demanded by the community, [roadmap](https://docs.kafka-ui.provectus.io/project/roadmap) plans, etc.). The final decision is left for the maintainers' discretion. All bugs should be confirmed as such (i.e. the behavior is unintended). Any issue should be properly triaged by the maintainers beforehand, which includes: 1. Having a proper milestone set 2. Having required labels assigned: accepted label, scope labels, etc. Formally, if these triage conditions are met, you can start to work on the issue. With all these requirements met, feel free to pick the issue you want. Reach out to the maintainers if you have any questions. ## Working on the issue Every issue “in-progress” needs to be assigned to a corresponding person. To keep the status of the issue clear to everyone, please keep the card's status updated ("project" card to the right of the issue should match the milestone’s name). ## Setting up a local development environment Please refer to [this guide](https://docs.kafka-ui.provectus.io/development/contributing). # Pull Requests ## Branch naming In order to keep branch names uniform and easy-to-understand, please use the following conventions for branch naming. Generally speaking, it is a good idea to add a group/type prefix to a branch; e.g., if you are working on a specific branch, you could name it `issues/xxx`. Here is a list of good examples:
`issues/123`
`feature/feature_name`
`bugfix/fix_thing`
## Code style Java: There is a file called `checkstyle.xml` in project root under `etc` directory.
You can import it into IntelliJ IDEA via Checkstyle plugin. ## Naming conventions REST paths should be written in **lowercase** and consist of **plural** nouns only.
Also, multiple words that are placed in a single path segment should be divided by a hyphen (`-`).
Query variable names should be formatted in `camelCase`. Model names should consist of **plural** nouns only and should be formatted in `camelCase` as well. ## Creating a PR When creating a PR please do the following: 1. In commit messages use these [closing keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword). 2. Link an issue(-s) via "linked issues" block. 3. Set the PR labels. Ensure that you set only the same set of labels that is present in the issue, and ignore yellow `status/` labels. 4. If the PR does not close any of the issues, the PR itself might need to have a milestone set. Reach out to the maintainers to consult. 5. Assign the PR to yourself. A PR assignee is someone whose goal is to get the PR merged. 6. Add reviewers. As a rule, reviewers' suggestions are pretty good; please use them. 7. Upon merging the PR, please use a meaningful commit message, task name should be fine in this case. ### Pull Request checklist 1. When composing a build, ensure that any install or build dependencies have been removed before the end of the layer. 2. Update the `README.md` with the details of changes made to the interface. This includes new environment variables, exposed ports, useful file locations, and container parameters. ## Reviewing a PR WIP ### Pull Request reviewer checklist WIP ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 CloudHut Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ ![UI for Apache Kafka logo](documentation/images/kafka-ui-logo.png) UI for Apache Kafka  ------------------ #### Versatile, fast and lightweight web UI for managing Apache Kafka® clusters. Built by developers, for developers.
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/provectus/kafka-ui/blob/master/LICENSE) ![UI for Apache Kafka Price Free](documentation/images/free-open-source.svg) [![Release version](https://img.shields.io/github/v/release/provectus/kafka-ui)](https://github.com/provectus/kafka-ui/releases) [![Chat with us](https://img.shields.io/discord/897805035122077716)](https://discord.gg/4DWzD7pGE5) [![Docker pulls](https://img.shields.io/docker/pulls/provectuslabs/kafka-ui)](https://hub.docker.com/r/provectuslabs/kafka-ui)

DOCS • QUICK START • COMMUNITY DISCORD
AWS Marketplace • ProductHunt

#### UI for Apache Kafka is a free, open-source web UI to monitor and manage Apache Kafka clusters. UI for Apache Kafka is a simple tool that makes your data flows observable, helps find and troubleshoot issues faster and deliver optimal performance. Its lightweight dashboard makes it easy to track key metrics of your Kafka clusters - Brokers, Topics, Partitions, Production, and Consumption. ### DISCLAIMER UI for Apache Kafka is a free tool built and supported by the open-source community. Curated by Provectus, it will remain free and open-source, without any paid features or subscription plans to be added in the future. Looking for the help of Kafka experts? Provectus can help you design, build, deploy, and manage Apache Kafka clusters and streaming applications. Discover [Professional Services for Apache Kafka](https://provectus.com/professional-services-apache-kafka/), to unlock the full potential of Kafka in your enterprise! Set up UI for Apache Kafka with just a couple of easy commands to visualize your Kafka data in a comprehensible way. You can run the tool locally or in the cloud. ![Interface](documentation/images/Interface.gif) # Features * **Multi-Cluster Management** — monitor and manage all your clusters in one place * **Performance Monitoring with Metrics Dashboard** — track key Kafka metrics with a lightweight dashboard * **View Kafka Brokers** — view topic and partition assignments, controller status * **View Kafka Topics** — view partition count, replication status, and custom configuration * **View Consumer Groups** — view per-partition parked offsets, combined and per-partition lag * **Browse Messages** — browse messages with JSON, plain text, and Avro encoding * **Dynamic Topic Configuration** — create and configure new topics with dynamic configuration * **Configurable Authentification** — [secure](https://docs.kafka-ui.provectus.io/configuration/authentication) your installation with optional Github/Gitlab/Google OAuth 2.0 * **Custom serialization/deserialization plugins** - [use](https://docs.kafka-ui.provectus.io/configuration/serialization-serde) a ready-to-go serde for your data like AWS Glue or Smile, or code your own! * **Role based access control** - [manage permissions](https://docs.kafka-ui.provectus.io/configuration/rbac-role-based-access-control) to access the UI with granular precision * **Data masking** - [obfuscate](https://docs.kafka-ui.provectus.io/configuration/data-masking) sensitive data in topic messages # The Interface UI for Apache Kafka wraps major functions of Apache Kafka with an intuitive user interface. ![Interface](documentation/images/Interface.gif) ## Topics UI for Apache Kafka makes it easy for you to create topics in your browser by several clicks, pasting your own parameters, and viewing topics in the list. ![Create Topic](documentation/images/Create_topic_kafka-ui.gif) It's possible to jump from connectors view to corresponding topics and from a topic to consumers (back and forth) for more convenient navigation. connectors, overview topic settings. ![Connector_Topic_Consumer](documentation/images/Connector_Topic_Consumer.gif) ### Messages Let's say we want to produce messages for our topic. With the UI for Apache Kafka we can send or write data/messages to the Kafka topics without effort by specifying parameters, and viewing messages in the list. ![Produce Message](documentation/images/Create_message_kafka-ui.gif) ## Schema registry There are 3 supported types of schemas: Avro®, JSON Schema, and Protobuf schemas. ![Create Schema Registry](documentation/images/Create_schema.gif) Before producing avro/protobuf encoded messages, you have to add a schema for the topic in Schema Registry. Now all these steps are easy to do with a few clicks in a user-friendly interface. ![Avro Schema Topic](documentation/images/Schema_Topic.gif) # Getting Started To run UI for Apache Kafka, you can use either a pre-built Docker image or build it (or a jar file) yourself. ## Quick start (Demo run) ``` docker run -it -p 8080:8080 -e DYNAMIC_CONFIG_ENABLED=true provectuslabs/kafka-ui ``` Then access the web UI at [http://localhost:8080](http://localhost:8080) The command is sufficient to try things out. When you're done trying things out, you can proceed with a [persistent installation](https://docs.kafka-ui.provectus.io/quick-start/persistent-start) ## Persistent installation ``` services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 environment: DYNAMIC_CONFIG_ENABLED: 'true' volumes: - ~/kui/config.yml:/etc/kafkaui/dynamic_config.yaml ``` Please refer to our [configuration](https://docs.kafka-ui.provectus.io/configuration/quick-start) page to proceed with further app configuration. ## Some useful configuration related links [Web UI Cluster Configuration Wizard](https://docs.kafka-ui.provectus.io/configuration/configuration-wizard) [Configuration file explanation](https://docs.kafka-ui.provectus.io/configuration/configuration-file) [Docker Compose examples](https://docs.kafka-ui.provectus.io/configuration/compose-examples) [Misc configuration properties](https://docs.kafka-ui.provectus.io/configuration/misc-configuration-properties) ## Helm charts [Quick start](https://docs.kafka-ui.provectus.io/configuration/helm-charts/quick-start) ## Building from sources [Quick start](https://docs.kafka-ui.provectus.io/development/building/prerequisites) with building ## Liveliness and readiness probes Liveliness and readiness endpoint is at `/actuator/health`.
Info endpoint (build info) is located at `/actuator/info`. # Configuration options All of the environment variables/config properties could be found [here](https://docs.kafka-ui.provectus.io/configuration/misc-configuration-properties). # Contributing Please refer to [contributing guide](https://docs.kafka-ui.provectus.io/development/contributing), we'll guide you from there. ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions Following versions of the project are currently being supported with security updates. | Version | Supported | | ------- | ------------------ | | 0.7.x | :white_check_mark: | | 0.6.x | :x: | | 0.5.x | :x: | | 0.4.x | :x: | | 0.3.x | :x: | | 0.2.x | :x: | | 0.1.x | :x: | ## Reporting a Vulnerability Please **DO NOT** file a publicly available github issues regarding security vulnerabilities. Send us details via email (maintainers.kafka-ui "at" provectus.com). Consider adding something like "security vulnerability report" in the title of an email. ================================================ FILE: documentation/compose/DOCKER_COMPOSE.md ================================================ # Descriptions of docker-compose configurations (*.yaml) 1. [kafka-ui.yaml](./kafka-ui.yaml) - Default configuration with 2 kafka clusters with two nodes of Schema Registry, one kafka-connect and a few dummy topics. 2. [kafka-ui-arm64.yaml](./kafka-ui-arm64.yaml) - Default configuration for ARM64(Mac M1) architecture with 1 kafka cluster without zookeeper with one node of Schema Registry, one kafka-connect and a few dummy topics. 3. [kafka-clusters-only.yaml](./kafka-clusters-only.yaml) - A configuration for development purposes, everything besides `kafka-ui` itself (to be run locally). 4. [kafka-ui-ssl.yml](./kafka-ssl.yml) - Connect to Kafka via TLS/SSL 5. [kafka-cluster-sr-auth.yaml](./kafka-cluster-sr-auth.yaml) - Schema registry with authentication. 6. [kafka-ui-auth-context.yaml](./kafka-ui-auth-context.yaml) - Basic (username/password) authentication with custom path (URL) (issue 861). 7. [e2e-tests.yaml](./e2e-tests.yaml) - Configuration with different connectors (github-source, s3, sink-activities, source-activities) and Ksql functionality. 8. [kafka-ui-jmx-secured.yml](./kafka-ui-jmx-secured.yml) - Kafka’s JMX with SSL and authentication. 9. [kafka-ui-reverse-proxy.yaml](./nginx-proxy.yaml) - An example for using the app behind a proxy (like nginx). 10. [kafka-ui-sasl.yaml](./kafka-ui-sasl.yaml) - SASL auth for Kafka. 11. [kafka-ui-traefik-proxy.yaml](./traefik-proxy.yaml) - Traefik specific proxy configuration. 12. [oauth-cognito.yaml](./oauth-cognito.yaml) - OAuth2 with Cognito 13. [kafka-ui-with-jmx-exporter.yaml](./kafka-ui-with-jmx-exporter.yaml) - A configuration with 2 kafka clusters with enabled prometheus jmx exporters instead of jmx. 14. [kafka-with-zookeeper.yaml](./kafka-with-zookeeper.yaml) - An example for using kafka with zookeeper ================================================ FILE: documentation/compose/connectors/github-source.json ================================================ { "name": "github-source", "config": { "connector.class": "io.confluent.connect.github.GithubSourceConnector", "confluent.topic.bootstrap.servers": "kafka0:29092, kafka1:29092", "confluent.topic.replication.factor": "1", "tasks.max": "1", "github.service.url": "https://api.github.com", "github.access.token": "", "github.repositories": "provectus/kafka-ui", "github.resources": "issues,commits,pull_requests", "github.since": "2019-01-01", "topic.name.pattern": "github-${resourceName}", "key.converter": "org.apache.kafka.connect.json.JsonConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085" } } ================================================ FILE: documentation/compose/connectors/s3-sink.json ================================================ { "name": "s3-sink", "config": { "connector.class": "io.confluent.connect.s3.S3SinkConnector", "topics": "github-issues, github-pull_requests, github-commits", "tasks.max": "1", "s3.region": "eu-central-1", "s3.bucket.name": "kafka-ui-s3-sink-connector", "s3.part.size": "5242880", "flush.size": "3", "storage.class": "io.confluent.connect.s3.storage.S3Storage", "format.class": "io.confluent.connect.s3.format.json.JsonFormat", "schema.generator.class": "io.confluent.connect.storage.hive.schema.DefaultSchemaGenerator", "partitioner.class": "io.confluent.connect.storage.partitioner.DefaultPartitioner", "schema.compatibility": "BACKWARD" } } ================================================ FILE: documentation/compose/connectors/sink-activities.json ================================================ { "name": "sink_postgres_activities", "config": { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "source-activities", "table.name.format": "sink_activities", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "io.confluent.connect.avro.AvroConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085", "auto.create": "true", "pk.mode": "record_value", "pk.fields": "id", "insert.mode": "upsert" } } ================================================ FILE: documentation/compose/connectors/source-activities.json ================================================ { "name": "source_postgres_activities", "config": { "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topic.prefix": "source-", "poll.interval.ms": 3600000, "table.whitelist": "public.activities", "mode": "bulk", "transforms": "extractkey", "transforms.extractkey.type": "org.apache.kafka.connect.transforms.ExtractField$Key", "transforms.extractkey.field": "id", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "io.confluent.connect.avro.AvroConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085" } } ================================================ FILE: documentation/compose/connectors/start.sh ================================================ #! /bin/bash while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' kafka-connect0:8083)" != "200" ]] do sleep 5 done echo "\n --------------Creating connectors..." for filename in /connectors/*.json; do curl -X POST -H "Content-Type: application/json" -d @$filename http://kafka-connect0:8083/connectors done ================================================ FILE: documentation/compose/data/message.json ================================================ {} ================================================ FILE: documentation/compose/data/proxy.conf ================================================ server { listen 80; server_name localhost; location /kafka-ui { # rewrite /kafka-ui/(.*) /$1 break; proxy_pass http://kafka-ui:8080; } } ================================================ FILE: documentation/compose/e2e-tests.yaml ================================================ --- version: '3.5' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 healthcheck: test: wget --no-verbose --tries=1 --spider http://localhost:8080/actuator/health interval: 30s timeout: 10s retries: 10 depends_on: kafka0: condition: service_healthy schemaregistry0: condition: service_healthy kafka-connect0: condition: service_healthy environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 KAFKA_CLUSTERS_0_KSQLDBSERVER: http://ksqldb:8088 kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 healthcheck: test: unset JMX_PORT && KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9999" && kafka-broker-api-versions --bootstrap-server=localhost:9092 interval: 30s timeout: 10s retries: 10 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 ports: - 8085:8085 depends_on: kafka0: condition: service_healthy healthcheck: test: [ "CMD", "timeout", "1", "curl", "--silent", "--fail", "http://schemaregistry0:8085/subjects" ] interval: 30s timeout: 10s retries: 10 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-connect0: build: context: ./kafka-connect args: image: confluentinc/cp-kafka-connect:6.0.1 ports: - 8083:8083 depends_on: kafka0: condition: service_healthy schemaregistry0: condition: service_healthy healthcheck: test: [ "CMD", "nc", "127.0.0.1", "8083" ] interval: 30s timeout: 10s retries: 10 environment: CONNECT_BOOTSTRAP_SERVERS: kafka0:29092 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 CONNECT_STATUS_STORAGE_TOPIC: _connect_status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" # AWS_ACCESS_KEY_ID: "" # AWS_SECRET_ACCESS_KEY: "" kafka-init-topics: image: confluentinc/cp-kafka:7.2.1 volumes: - ./data/message.json:/data/message.json depends_on: kafka0: condition: service_healthy command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka0:29092 1 30 && \ kafka-topics --create --topic users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-topics --create --topic messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-console-producer --bootstrap-server kafka0:29092 --topic users < /data/message.json'" postgres-db: build: context: ./postgres args: image: postgres:9.6.22 ports: - 5432:5432 healthcheck: test: [ "CMD-SHELL", "pg_isready -U dev_user" ] interval: 10s timeout: 5s retries: 5 environment: POSTGRES_USER: 'dev_user' POSTGRES_PASSWORD: '12345' create-connectors: image: ellerbrock/alpine-bash-curl-ssl depends_on: postgres-db: condition: service_healthy kafka-connect0: condition: service_healthy volumes: - ./connectors:/connectors command: bash -c '/connectors/start.sh' ksqldb: image: confluentinc/ksqldb-server:0.18.0 healthcheck: test: [ "CMD", "timeout", "1", "curl", "--silent", "--fail", "http://localhost:8088/info" ] interval: 30s timeout: 10s retries: 10 depends_on: kafka0: condition: service_healthy kafka-connect0: condition: service_healthy schemaregistry0: condition: service_healthy ports: - 8088:8088 environment: KSQL_CUB_KAFKA_TIMEOUT: 120 KSQL_LISTENERS: http://0.0.0.0:8088 KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" KSQL_KSQL_CONNECT_URL: http://kafka-connect0:8083 KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 KSQL_KSQL_SERVICE_ID: my_ksql_1 KSQL_KSQL_HIDDEN_TOPICS: '^_.*' KSQL_CACHE_MAX_BYTES_BUFFERING: 0 ================================================ FILE: documentation/compose/jaas/client.properties ================================================ sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret"; security.protocol=SASL_PLAINTEXT sasl.mechanism=PLAIN ================================================ FILE: documentation/compose/jaas/kafka_connect.jaas ================================================ KafkaConnect { org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required file="/conf/kafka_connect.password"; }; ================================================ FILE: documentation/compose/jaas/kafka_connect.password ================================================ admin: admin-secret ================================================ FILE: documentation/compose/jaas/kafka_server.conf ================================================ KafkaServer { org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret" user_enzo="cisternino"; }; KafkaClient { org.apache.kafka.common.security.plain.PlainLoginModule required user_admin="admin-secret"; }; Client { org.apache.zookeeper.server.auth.DigestLoginModule required username="zkuser" password="zkuserpassword"; }; ================================================ FILE: documentation/compose/jaas/schema_registry.jaas ================================================ SchemaRegistryProps { org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required file="/conf/schema_registry.password" debug="false"; }; ================================================ FILE: documentation/compose/jaas/schema_registry.password ================================================ admin: OBF:1w8t1tvf1w261w8v1w1c1tvn1w8x,admin ================================================ FILE: documentation/compose/jaas/zookeeper_jaas.conf ================================================ Server { org.apache.zookeeper.server.auth.DigestLoginModule required user_zkuser="zkuserpassword"; }; ================================================ FILE: documentation/compose/jmx/jmxremote.access ================================================ root readwrite ================================================ FILE: documentation/compose/jmx/jmxremote.password ================================================ root password ================================================ FILE: documentation/compose/jmx-exporter/kafka-broker.yml ================================================ rules: - pattern: ".*" ================================================ FILE: documentation/compose/jmx-exporter/kafka-prepare-and-run ================================================ #!/usr/bin/env bash JAVA_AGENT_FILE="/usr/share/jmx_exporter/jmx_prometheus_javaagent.jar" if [ ! -f "$JAVA_AGENT_FILE" ] then echo "Downloading jmx_exporter javaagent" curl -o $JAVA_AGENT_FILE https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar fi exec /etc/confluent/docker/run ================================================ FILE: documentation/compose/kafka-cluster-sr-auth.yaml ================================================ --- version: '2' services: kafka1: image: confluentinc/cp-kafka:7.2.1 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka1:29092,CONTROLLER://kafka1:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry1: image: confluentinc/cp-schema-registry:7.2.1 ports: - 18085:8085 depends_on: - kafka1 volumes: - ./jaas:/conf environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry1 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry1:8085 # Default credentials: admin/letmein SCHEMA_REGISTRY_AUTHENTICATION_METHOD: BASIC SCHEMA_REGISTRY_AUTHENTICATION_REALM: SchemaRegistryProps SCHEMA_REGISTRY_AUTHENTICATION_ROLES: admin SCHEMA_REGISTRY_OPTS: -Djava.security.auth.login.config=/conf/schema_registry.jaas SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-init-topics: image: confluentinc/cp-kafka:7.2.1 volumes: - ./data/message.json:/data/message.json depends_on: - kafka1 command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka1:29092 1 30 && \ kafka-topics --create --topic users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka1:29092 && \ kafka-topics --create --topic messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka1:29092 && \ kafka-console-producer --bootstrap-server kafka1:29092 --topic users < /data/message.json'" kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka1 - schemaregistry1 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka1:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry1:8085 KAFKA_CLUSTERS_0_SCHEMAREGISTRYAUTH_USERNAME: admin KAFKA_CLUSTERS_0_SCHEMAREGISTRYAUTH_PASSWORD: letmein ================================================ FILE: documentation/compose/kafka-connect/Dockerfile ================================================ ARG image FROM ${image} ## Install connectors RUN echo "\nInstalling all required connectors...\n" && \ confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:latest && \ confluent-hub install --no-prompt confluentinc/kafka-connect-github:latest && \ confluent-hub install --no-prompt confluentinc/kafka-connect-s3:latest ================================================ FILE: documentation/compose/kafka-ssl-components.yaml ================================================ --- version: '3.4' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - schemaregistry0 - kafka-connect0 - ksqldb0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SSL KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 # SSL LISTENER! KAFKA_CLUSTERS_0_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' # DISABLE COMMON NAME VERIFICATION KAFKA_CLUSTERS_0_SCHEMAREGISTRY: https://schemaregistry0:8085 KAFKA_CLUSTERS_0_SCHEMAREGISTRYSSL_KEYSTORELOCATION: /kafka.keystore.jks KAFKA_CLUSTERS_0_SCHEMAREGISTRYSSL_KEYSTOREPASSWORD: "secret" KAFKA_CLUSTERS_0_KSQLDBSERVER: https://ksqldb0:8088 KAFKA_CLUSTERS_0_KSQLDBSERVERSSL_KEYSTORELOCATION: /kafka.keystore.jks KAFKA_CLUSTERS_0_KSQLDBSERVERSSL_KEYSTOREPASSWORD: "secret" KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: local KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: https://kafka-connect0:8083 KAFKA_CLUSTERS_0_KAFKACONNECT_0_KEYSTORELOCATION: /kafka.keystore.jks KAFKA_CLUSTERS_0_KAFKACONNECT_0_KEYSTOREPASSWORD: "secret" KAFKA_CLUSTERS_0_SSL_TRUSTSTORELOCATION: /kafka.truststore.jks KAFKA_CLUSTERS_0_SSL_TRUSTSTOREPASSWORD: "secret" DYNAMIC_CONFIG_ENABLED: 'true' # not necessary for ssl, added for tests volumes: - ./ssl/kafka.truststore.jks:/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/kafka.keystore.jks kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SSL:SSL,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'SSL://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'SSL://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' KAFKA_SECURITY_PROTOCOL: SSL KAFKA_SSL_ENABLED_MECHANISMS: PLAIN,SSL KAFKA_SSL_KEYSTORE_FILENAME: kafka.keystore.jks KAFKA_SSL_KEYSTORE_CREDENTIALS: creds KAFKA_SSL_KEY_CREDENTIALS: creds KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_CREDENTIALS: creds #KAFKA_SSL_CLIENT_AUTH: 'required' KAFKA_SSL_CLIENT_AUTH: 'requested' KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' # COMMON NAME VERIFICATION IS DISABLED SERVER-SIDE volumes: - ./scripts/update_run.sh:/tmp/update_run.sh - ./ssl/creds:/etc/kafka/secrets/creds - ./ssl/kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/etc/kafka/secrets/kafka.keystore.jks command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: SSL://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SSL SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_LOCATION: /kafka.truststore.jks SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_PASSWORD: secret SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_LOCATION: /kafka.keystore.jks SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_PASSWORD: secret SCHEMA_REGISTRY_KAFKASTORE_SSL_KEY_PASSWORD: secret SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: https://schemaregistry0:8085 SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: https SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "https" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas SCHEMA_REGISTRY_SSL_CLIENT_AUTHENTICATION: "REQUIRED" SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION: /kafka.truststore.jks SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: secret SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION: /kafka.keystore.jks SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: secret SCHEMA_REGISTRY_SSL_KEY_PASSWORD: secret ports: - 8085:8085 volumes: - ./ssl/kafka.truststore.jks:/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/kafka.keystore.jks kafka-connect0: image: confluentinc/cp-kafka-connect:7.2.1 ports: - 8083:8083 depends_on: - kafka0 - schemaregistry0 environment: CONNECT_BOOTSTRAP_SERVERS: kafka0:29092 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 CONNECT_STATUS_STORAGE_TOPIC: _connect_status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: https://schemaregistry0:8085 CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: https://schemaregistry0:8085 CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" CONNECT_SECURITY_PROTOCOL: "SSL" CONNECT_SSL_KEYSTORE_LOCATION: "/kafka.keystore.jks" CONNECT_SSL_KEY_PASSWORD: "secret" CONNECT_SSL_KEYSTORE_PASSWORD: "secret" CONNECT_SSL_TRUSTSTORE_LOCATION: "/kafka.truststore.jks" CONNECT_SSL_TRUSTSTORE_PASSWORD: "secret" CONNECT_SSL_CLIENT_AUTH: "requested" CONNECT_REST_ADVERTISED_LISTENER: "https" CONNECT_LISTENERS: "https://kafka-connect0:8083" volumes: - ./ssl/kafka.truststore.jks:/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/kafka.keystore.jks ksqldb0: image: confluentinc/ksqldb-server:0.18.0 depends_on: - kafka0 - kafka-connect0 - schemaregistry0 ports: - 8088:8088 environment: KSQL_CUB_KAFKA_TIMEOUT: 120 KSQL_LISTENERS: https://0.0.0.0:8088 KSQL_BOOTSTRAP_SERVERS: SSL://kafka0:29092 KSQL_SECURITY_PROTOCOL: SSL KSQL_SSL_TRUSTSTORE_LOCATION: /kafka.truststore.jks KSQL_SSL_TRUSTSTORE_PASSWORD: secret KSQL_SSL_KEYSTORE_LOCATION: /kafka.keystore.jks KSQL_SSL_KEYSTORE_PASSWORD: secret KSQL_SSL_KEY_PASSWORD: secret KSQL_SSL_CLIENT_AUTHENTICATION: REQUIRED KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true" KSQL_KSQL_CONNECT_URL: https://kafka-connect0:8083 KSQL_KSQL_SCHEMA_REGISTRY_URL: https://schemaregistry0:8085 KSQL_KSQL_SERVICE_ID: my_ksql_1 KSQL_KSQL_HIDDEN_TOPICS: '^_.*' KSQL_CACHE_MAX_BYTES_BUFFERING: 0 volumes: - ./ssl/kafka.truststore.jks:/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/kafka.keystore.jks ================================================ FILE: documentation/compose/kafka-ssl.yml ================================================ --- version: '3.4' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SSL KAFKA_CLUSTERS_0_PROPERTIES_SSL_KEYSTORE_LOCATION: /kafka.keystore.jks KAFKA_CLUSTERS_0_PROPERTIES_SSL_KEYSTORE_PASSWORD: "secret" KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092 # SSL LISTENER! KAFKA_CLUSTERS_0_SSL_TRUSTSTORELOCATION: /kafka.truststore.jks KAFKA_CLUSTERS_0_SSL_TRUSTSTOREPASSWORD: "secret" KAFKA_CLUSTERS_0_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' # DISABLE COMMON NAME VERIFICATION volumes: - ./ssl/kafka.truststore.jks:/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/kafka.keystore.jks kafka: image: confluentinc/cp-kafka:7.2.1 hostname: kafka container_name: kafka ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SSL:SSL,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'SSL://kafka:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' KAFKA_LISTENERS: 'SSL://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' KAFKA_SECURITY_PROTOCOL: SSL KAFKA_SSL_ENABLED_MECHANISMS: PLAIN,SSL KAFKA_SSL_KEYSTORE_FILENAME: kafka.keystore.jks KAFKA_SSL_KEYSTORE_CREDENTIALS: creds KAFKA_SSL_KEY_CREDENTIALS: creds KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_CREDENTIALS: creds #KAFKA_SSL_CLIENT_AUTH: 'required' KAFKA_SSL_CLIENT_AUTH: 'requested' KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' # COMMON NAME VERIFICATION IS DISABLED SERVER-SIDE volumes: - ./scripts/update_run.sh:/tmp/update_run.sh - ./ssl/creds:/etc/kafka/secrets/creds - ./ssl/kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks - ./ssl/kafka.keystore.jks:/etc/kafka/secrets/kafka.keystore.jks command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" ================================================ FILE: documentation/compose/kafka-ui-acl-with-zk.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - zookeeper - kafka environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092 KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM: PLAIN KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";' zookeeper: image: wurstmeister/zookeeper:3.4.6 environment: JVMFLAGS: "-Djava.security.auth.login.config=/etc/zookeeper/zookeeper_jaas.conf" volumes: - ./jaas/zookeeper_jaas.conf:/etc/zookeeper/zookeeper_jaas.conf ports: - 2181:2181 kafka: image: confluentinc/cp-kafka:7.2.1 hostname: kafka container_name: kafka ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server.conf" KAFKA_AUTHORIZER_CLASS_NAME: "kafka.security.authorizer.AclAuthorizer" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' KAFKA_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'SASL_PLAINTEXT' KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN' KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' KAFKA_SECURITY_PROTOCOL: 'SASL_PLAINTEXT' KAFKA_SUPER_USERS: 'User:admin' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh - ./jaas:/etc/kafka/jaas ================================================ FILE: documentation/compose/kafka-ui-arm64.yaml ================================================ # ARM64 supported images for kafka can be found here # https://hub.docker.com/r/confluentinc/cp-kafka/tags?page=1&name=arm64 --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - schema-registry0 - kafka-connect0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 DYNAMIC_CONFIG_ENABLED: 'true' # not necessary, added for tests KAFKA_CLUSTERS_0_AUDIT_TOPICAUDITENABLED: 'true' KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true' kafka0: image: confluentinc/cp-kafka:7.2.1.arm64 hostname: kafka0 container_name: kafka0 ports: - 9092:9092 - 9997:9997 environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092 KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' KAFKA_JMX_PORT: 9997 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schema-registry0: image: confluentinc/cp-schema-registry:7.2.1.arm64 ports: - 8085:8085 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schema-registry0 SCHEMA_REGISTRY_LISTENERS: http://schema-registry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-connect0: image: confluentinc/cp-kafka-connect:7.2.1.arm64 ports: - 8083:8083 depends_on: - kafka0 - schema-registry0 environment: CONNECT_BOOTSTRAP_SERVERS: kafka0:29092 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 CONNECT_STATUS_STORAGE_TOPIC: _connect_status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry0:8085 CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry0:8085 CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" kafka-init-topics: image: confluentinc/cp-kafka:7.2.1.arm64 volumes: - ./data/message.json:/data/message.json depends_on: - kafka0 command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka0:29092 1 30 && \ kafka-topics --create --topic second.users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-topics --create --topic second.messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-topics --create --topic first.messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-console-producer --bootstrap-server kafka0:29092 --topic second.users < /data/message.json'" ================================================ FILE: documentation/compose/kafka-ui-auth-context.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 SERVER_SERVLET_CONTEXT_PATH: /kafkaui AUTH_TYPE: "LOGIN_FORM" SPRING_SECURITY_USER_NAME: admin SPRING_SECURITY_USER_PASSWORD: pass kafka: image: confluentinc/cp-kafka:7.2.1 hostname: kafka container_name: kafka ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" ================================================ FILE: documentation/compose/kafka-ui-connectors-auth.yaml ================================================ --- version: "2" services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - schemaregistry0 - kafka-connect0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 KAFKA_CLUSTERS_0_KAFKACONNECT_0_USERNAME: admin KAFKA_CLUSTERS_0_KAFKACONNECT_0_PASSWORD: admin-secret kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 KAFKA_PROCESS_ROLES: "broker,controller" KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka0:29093" KAFKA_LISTENERS: "PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092" KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs" volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: 'bash -c ''if [ ! -f /tmp/update_run.sh ]; then echo "ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi''' schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 ports: - 8085:8085 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-connect0: build: context: ./kafka-connect args: image: confluentinc/cp-kafka-connect:7.2.1 ports: - 8083:8083 depends_on: - kafka0 - schemaregistry0 volumes: - ./jaas:/conf environment: CONNECT_BOOTSTRAP_SERVERS: kafka0:29092 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 CONNECT_STATUS_STORAGE_TOPIC: _connect_status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_REST_PORT: 8083 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" CONNECT_REST_EXTENSION_CLASSES: "org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension" KAFKA_OPTS: "-Djava.security.auth.login.config=/conf/kafka_connect.jaas" # AWS_ACCESS_KEY_ID: "" # AWS_SECRET_ACCESS_KEY: "" kafka-init-topics: image: confluentinc/cp-kafka:7.2.1 volumes: - ./data/message.json:/data/message.json depends_on: - kafka0 command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka0:29092 1 30 && \ kafka-topics --create --topic users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-topics --create --topic messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-console-producer --bootstrap-server kafka0:29092 --topic users < /data/message.json'" ================================================ FILE: documentation/compose/kafka-ui-jmx-secured.yml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_METRICS_USERNAME: root KAFKA_CLUSTERS_0_METRICS_PASSWORD: password KAFKA_CLUSTERS_0_METRICS_KEYSTORE_LOCATION: /jmx/clientkeystore KAFKA_CLUSTERS_0_METRICS_KEYSTORE_PASSWORD: '12345678' KAFKA_CLUSTERS_0_SSL_TRUSTSTORE_LOCATION: /jmx/clienttruststore KAFKA_CLUSTERS_0_SSL_TRUSTSTORE_PASSWORD: '12345678' volumes: - ./jmx/clienttruststore:/jmx/clienttruststore - ./jmx/clientkeystore:/jmx/clientkeystore kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - 9092:9092 - 9997:9997 environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' # CHMOD 700 FOR JMXREMOTE.* FILES KAFKA_JMX_OPTS: >- -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.ssl=true -Dcom.sun.management.jmxremote.registry.ssl=true -Dcom.sun.management.jmxremote.ssl.need.client.auth=true -Djavax.net.ssl.keyStore=/jmx/serverkeystore -Djavax.net.ssl.keyStorePassword=12345678 -Djavax.net.ssl.trustStore=/jmx/servertruststore -Djavax.net.ssl.trustStorePassword=12345678 -Dcom.sun.management.jmxremote.password.file=/jmx/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/jmx/jmxremote.access -Dcom.sun.management.jmxremote.rmi.port=9997 -Djava.rmi.server.hostname=kafka0 volumes: - ./jmx/serverkeystore:/jmx/serverkeystore - ./jmx/servertruststore:/jmx/servertruststore - ./jmx/jmxremote.password:/jmx/jmxremote.password - ./jmx/jmxremote.access:/jmx/jmxremote.access - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" ================================================ FILE: documentation/compose/kafka-ui-sasl.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092 KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SASL_PLAINTEXT KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM: PLAIN KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret";' DYNAMIC_CONFIG_ENABLED: true # not necessary for sasl auth, added for tests kafka: image: confluentinc/cp-kafka:7.2.1 hostname: kafka container_name: kafka ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server.conf" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' KAFKA_LISTENERS: 'SASL_PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'SASL_PLAINTEXT' KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN' KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' KAFKA_SECURITY_PROTOCOL: 'SASL_PLAINTEXT' KAFKA_SUPER_USERS: 'User:admin,User:enzo' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh - ./jaas:/etc/kafka/jaas command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" ================================================ FILE: documentation/compose/kafka-ui-serdes.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - schemaregistry0 environment: kafka.clusters.0.name: SerdeExampleCluster kafka.clusters.0.bootstrapServers: kafka0:29092 kafka.clusters.0.schemaRegistry: http://schemaregistry0:8085 # optional SSL settings for cluster (will be used by SchemaRegistry serde, if set) #kafka.clusters.0.ssl.keystoreLocation: /kafka.keystore.jks #kafka.clusters.0.ssl.keystorePassword: "secret" #kafka.clusters.0.ssl.truststoreLocation: /kafka.truststore.jks #kafka.clusters.0.ssl.truststorePassword: "secret" # optional auth properties for SR #kafka.clusters.0.schemaRegistryAuth.username: "use" #kafka.clusters.0.schemaRegistryAuth.password: "pswrd" kafka.clusters.0.defaultKeySerde: Int32 #optional kafka.clusters.0.defaultValueSerde: String #optional kafka.clusters.0.serde.0.name: ProtobufFile kafka.clusters.0.serde.0.topicKeysPattern: "topic1" kafka.clusters.0.serde.0.topicValuesPattern: "topic1" kafka.clusters.0.serde.0.properties.protobufFilesDir: /protofiles/ kafka.clusters.0.serde.0.properties.protobufMessageNameForKey: test.MyKey # default type for keys kafka.clusters.0.serde.0.properties.protobufMessageName: test.MyValue # default type for values kafka.clusters.0.serde.0.properties.protobufMessageNameForKeyByTopic.topic1: test.MySpecificTopicKey # keys type for topic "topic1" kafka.clusters.0.serde.0.properties.protobufMessageNameByTopic.topic1: test.MySpecificTopicValue # values type for topic "topic1" kafka.clusters.0.serde.1.name: String #kafka.clusters.0.serde.1.properties.encoding: "UTF-16" #optional, default is UTF-8 kafka.clusters.0.serde.1.topicValuesPattern: "json-events|text-events" kafka.clusters.0.serde.2.name: AsciiString kafka.clusters.0.serde.2.className: com.provectus.kafka.ui.serdes.builtin.StringSerde kafka.clusters.0.serde.2.properties.encoding: "ASCII" kafka.clusters.0.serde.3.name: SchemaRegistry # will be configured automatically using cluster SR kafka.clusters.0.serde.3.topicValuesPattern: "sr-topic.*" kafka.clusters.0.serde.4.name: AnotherSchemaRegistry kafka.clusters.0.serde.4.className: com.provectus.kafka.ui.serdes.builtin.sr.SchemaRegistrySerde kafka.clusters.0.serde.4.properties.url: http://schemaregistry0:8085 kafka.clusters.0.serde.4.properties.keySchemaNameTemplate: "%s-key" kafka.clusters.0.serde.4.properties.schemaNameTemplate: "%s-value" #kafka.clusters.0.serde.4.topicValuesPattern: "sr2-topic.*" # optional auth and ssl properties for SR (overrides cluster-level): #kafka.clusters.0.serde.4.properties.username: "user" #kafka.clusters.0.serde.4.properties.password: "passw" #kafka.clusters.0.serde.4.properties.keystoreLocation: /kafka.keystore.jks #kafka.clusters.0.serde.4.properties.keystorePassword: "secret" #kafka.clusters.0.serde.4.properties.truststoreLocation: /kafka.truststore.jks #kafka.clusters.0.serde.4.properties.truststorePassword: "secret" kafka.clusters.0.serde.5.name: UInt64 kafka.clusters.0.serde.5.topicKeysPattern: "topic-with-uint64keys" volumes: - ./proto:/protofiles kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 ports: - 8085:8085 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas ================================================ FILE: documentation/compose/kafka-ui-with-jmx-exporter.yaml ================================================ --- version: '2' services: kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "11001:11001" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' KAFKA_OPTS: -javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent.jar=11001:/usr/share/jmx_exporter/kafka-broker.yml volumes: - ./jmx-exporter:/usr/share/jmx_exporter/ - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /usr/share/jmx_exporter/kafka-prepare-and-run ; fi'" kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 11001 KAFKA_CLUSTERS_0_METRICS_TYPE: PROMETHEUS ================================================ FILE: documentation/compose/kafka-ui.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - kafka1 - schemaregistry0 - schemaregistry1 - kafka-connect0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085 KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083 KAFKA_CLUSTERS_1_NAME: secondLocal KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS: kafka1:29092 KAFKA_CLUSTERS_1_METRICS_PORT: 9998 KAFKA_CLUSTERS_1_SCHEMAREGISTRY: http://schemaregistry1:8085 DYNAMIC_CONFIG_ENABLED: 'true' kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" kafka1: image: confluentinc/cp-kafka:7.2.1 hostname: kafka1 container_name: kafka1 ports: - "9093:9092" - "9998:9998" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9998 KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9998 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka1:29092,CONTROLLER://kafka1:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 ports: - 8085:8085 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas schemaregistry1: image: confluentinc/cp-schema-registry:7.2.1 ports: - 18085:8085 depends_on: - kafka1 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry1 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry1:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-connect0: image: confluentinc/cp-kafka-connect:7.2.1 ports: - 8083:8083 depends_on: - kafka0 - schemaregistry0 environment: CONNECT_BOOTSTRAP_SERVERS: kafka0:29092 CONNECT_GROUP_ID: compose-connect-group CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 CONNECT_STATUS_STORAGE_TOPIC: _connect_status CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085 CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0 CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" kafka-init-topics: image: confluentinc/cp-kafka:7.2.1 volumes: - ./data/message.json:/data/message.json depends_on: - kafka1 command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka1:29092 1 30 && \ kafka-topics --create --topic second.users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka1:29092 && \ kafka-topics --create --topic second.messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka1:29092 && \ kafka-topics --create --topic first.messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka0:29092 && \ kafka-console-producer --bootstrap-server kafka1:29092 -topic second.users < /data/message.json'" ================================================ FILE: documentation/compose/kafka-with-zookeeper.yaml ================================================ --- version: '2' services: zookeeper: image: confluentinc/cp-zookeeper:7.2.1 hostname: zookeeper container_name: zookeeper ports: - "2181:2181" environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 kafka: image: confluentinc/cp-server:7.2.1 hostname: kafka container_name: kafka depends_on: - zookeeper ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: kafka kafka-init-topics: image: confluentinc/cp-kafka:7.2.1 volumes: - ./data/message.json:/data/message.json depends_on: - kafka command: "bash -c 'echo Waiting for Kafka to be ready... && \ cub kafka-ready -b kafka:29092 1 30 && \ kafka-topics --create --topic users --partitions 3 --replication-factor 1 --if-not-exists --bootstrap-server kafka:29092 && \ kafka-topics --create --topic messages --partitions 2 --replication-factor 1 --if-not-exists --bootstrap-server kafka:29092 && \ kafka-console-producer --bootstrap-server kafka:29092 --topic users < /data/message.json'" ================================================ FILE: documentation/compose/ldap.yaml ================================================ --- version: '2' services: kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8080:8080 depends_on: - kafka0 - schemaregistry0 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 KAFKA_CLUSTERS_0_METRICS_PORT: 9997 KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085 AUTH_TYPE: "LDAP" SPRING_LDAP_URLS: "ldap://ldap:10389" SPRING_LDAP_BASE: "cn={0},ou=people,dc=planetexpress,dc=com" SPRING_LDAP_ADMIN_USER: "cn=admin,dc=planetexpress,dc=com" SPRING_LDAP_ADMIN_PASSWORD: "GoodNewsEveryone" SPRING_LDAP_USER_FILTER_SEARCH_BASE: "dc=planetexpress,dc=com" SPRING_LDAP_USER_FILTER_SEARCH_FILTER: "(&(uid={0})(objectClass=inetOrgPerson))" SPRING_LDAP_GROUP_FILTER_SEARCH_BASE: "ou=people,dc=planetexpress,dc=com" # OAUTH2.LDAP.ACTIVEDIRECTORY: true # OAUTH2.LDAP.AСTIVEDIRECTORY.DOMAIN: "memelord.lol" ldap: image: rroemhild/test-openldap:latest hostname: "ldap" ports: - 10389:10389 kafka0: image: confluentinc/cp-kafka:7.2.1 hostname: kafka0 container_name: kafka0 ports: - "9092:9092" - "9997:9997" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9997 KAFKA_JMX_HOSTNAME: localhost KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_NODE_ID: 1 KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka0:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka0:29092,CONTROLLER://kafka0:29093,PLAINTEXT_HOST://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' volumes: - ./scripts/update_run.sh:/tmp/update_run.sh command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'" schemaregistry0: image: confluentinc/cp-schema-registry:7.2.1 ports: - 8085:8085 depends_on: - kafka0 environment: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092 SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT SCHEMA_REGISTRY_HOST_NAME: schemaregistry0 SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085 SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http" SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas ================================================ FILE: documentation/compose/nginx-proxy.yaml ================================================ --- version: '2' services: nginx: image: nginx:latest volumes: - ./data/proxy.conf:/etc/nginx/conf.d/default.conf ports: - 8080:80 kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8082:8080 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 SERVER_SERVLET_CONTEXT_PATH: /kafka-ui ================================================ FILE: documentation/compose/postgres/Dockerfile ================================================ ARG image FROM ${image} MAINTAINER Provectus Team ADD data.sql /docker-entrypoint-initdb.d EXPOSE 5432 ================================================ FILE: documentation/compose/postgres/data.sql ================================================ CREATE DATABASE test WITH OWNER = dev_user; \connect test CREATE TABLE activities ( id INTEGER PRIMARY KEY, msg varchar(24), action varchar(128), browser varchar(24), device json, createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ); insert into activities(id, action, msg, browser, device) values (1, 'LOGIN', 'Success', 'Chrome', '{ "name": "Chrome", "major": "67", "version": "67.0.3396.99" }'), (2, 'LOGIN', 'Failed', 'Apple WebKit', '{ "name": "WebKit", "major": "605", "version": "605.1.15" }'); ================================================ FILE: documentation/compose/proto/key-types.proto ================================================ syntax = "proto3"; package test; import "google/protobuf/wrappers.proto"; message MyKey { string myKeyF1 = 1; google.protobuf.UInt64Value uint_64_wrapper = 2; } message MySpecificTopicKey { string special_field1 = 1; string special_field2 = 2; google.protobuf.FloatValue float_wrapper = 3; } ================================================ FILE: documentation/compose/proto/values.proto ================================================ syntax = "proto3"; package test; message MySpecificTopicValue { string f1 = 1; string f2 = 2; } message MyValue { int32 version = 1; string payload = 2; map intToStringMap = 3; map strToObjMap = 4; } ================================================ FILE: documentation/compose/scripts/clusterID ================================================ zlFiTJelTOuhnklFwLWixw ================================================ FILE: documentation/compose/scripts/create_cluster_id.sh ================================================ kafka-storage random-uuid > /workspace/kafka-ui/documentation/compose/clusterID ================================================ FILE: documentation/compose/scripts/update_run.sh ================================================ # This script is required to run kafka cluster (without zookeeper) #!/bin/sh # Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure # Docker workaround: Ignore cub zk-ready sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure # KRaft required step: Format the storage directory with a new cluster ID echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c /etc/kafka/kafka.properties" >> /etc/confluent/docker/ensure ================================================ FILE: documentation/compose/scripts/update_run_cluster.sh ================================================ # This script is required to run kafka cluster (without zookeeper) #!/bin/sh # Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure # Docker workaround: Ignore cub zk-ready sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure # KRaft required step: Format the storage directory with a new cluster ID echo "kafka-storage format --ignore-formatted -t $(cat /tmp/clusterID) -c /etc/kafka/kafka.properties" >> /etc/confluent/docker/ensure ================================================ FILE: documentation/compose/ssl/creds ================================================ secret ================================================ FILE: documentation/compose/ssl/generate_certs.sh ================================================ #!/usr/bin/env bash set -eu KEYSTORE_FILENAME="kafka.keystore.jks" VALIDITY_IN_DAYS=3650 DEFAULT_TRUSTSTORE_FILENAME="kafka.truststore.jks" TRUSTSTORE_WORKING_DIRECTORY="truststore" KEYSTORE_WORKING_DIRECTORY="keystore" CA_CERT_FILE="ca-cert" KEYSTORE_SIGN_REQUEST="cert-file" KEYSTORE_SIGN_REQUEST_SRL="ca-cert.srl" KEYSTORE_SIGNED_CERT="cert-signed" export COUNTRY=US export STATE=IL export ORGANIZATION_UNIT=SE export CITY=Chicago export PASSWORD=secret COUNTRY=$COUNTRY STATE=$STATE OU=$ORGANIZATION_UNIT CN=kafka0 # COMMON NAME VERIFICATION GOES BRR LOCATION=$CITY PASS=$PASSWORD function file_exists_and_exit() { echo "'$1' cannot exist. Move or delete it before" echo "re-running this script." exit 1 } if [ -e "$KEYSTORE_WORKING_DIRECTORY" ]; then file_exists_and_exit $KEYSTORE_WORKING_DIRECTORY fi if [ -e "$CA_CERT_FILE" ]; then file_exists_and_exit $CA_CERT_FILE fi if [ -e "$KEYSTORE_SIGN_REQUEST" ]; then file_exists_and_exit $KEYSTORE_SIGN_REQUEST fi if [ -e "$KEYSTORE_SIGN_REQUEST_SRL" ]; then file_exists_and_exit $KEYSTORE_SIGN_REQUEST_SRL fi if [ -e "$KEYSTORE_SIGNED_CERT" ]; then file_exists_and_exit $KEYSTORE_SIGNED_CERT fi echo "Welcome to the Kafka SSL keystore and trust store generator script." trust_store_file="" trust_store_private_key_file="" if [ -e "$TRUSTSTORE_WORKING_DIRECTORY" ]; then file_exists_and_exit $TRUSTSTORE_WORKING_DIRECTORY fi mkdir $TRUSTSTORE_WORKING_DIRECTORY echo echo "OK, we'll generate a trust store and associated private key." echo echo "First, the private key." echo openssl req -new -x509 -keyout $TRUSTSTORE_WORKING_DIRECTORY/ca-key \ -out $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -days $VALIDITY_IN_DAYS -nodes \ -subj "/C=$COUNTRY/ST=$STATE/L=$LOCATION/O=$OU/CN=$CN" trust_store_private_key_file="$TRUSTSTORE_WORKING_DIRECTORY/ca-key" echo echo "Two files were created:" echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-key -- the private key used later to" echo " sign certificates" echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -- the certificate that will be" echo " stored in the trust store in a moment and serve as the certificate" echo " authority (CA). Once this certificate has been stored in the trust" echo " store, it will be deleted. It can be retrieved from the trust store via:" echo " $ keytool -keystore -export -alias CARoot -rfc" echo echo "Now the trust store will be generated from the certificate." echo keytool -keystore $TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME \ -alias CARoot -import -file $TRUSTSTORE_WORKING_DIRECTORY/ca-cert \ -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS trust_store_file="$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME" echo echo "$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME was created." # don't need the cert because it's in the trust store. rm $TRUSTSTORE_WORKING_DIRECTORY/$CA_CERT_FILE echo echo "Continuing with:" echo " - trust store file: $trust_store_file" echo " - trust store private key: $trust_store_private_key_file" mkdir $KEYSTORE_WORKING_DIRECTORY echo echo "Now, a keystore will be generated. Each broker and logical client needs its own" echo "keystore. This script will create only one keystore. Run this script multiple" echo "times for multiple keystores." echo echo " NOTE: currently in Kafka, the Common Name (CN) does not need to be the FQDN of" echo " this host. However, at some point, this may change. As such, make the CN" echo " the FQDN. Some operating systems call the CN prompt 'first / last name'" # To learn more about CNs and FQDNs, read: # https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/X509ExtendedTrustManager.html keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME \ -alias localhost -validity $VALIDITY_IN_DAYS -genkey -keyalg RSA \ -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS echo echo "'$KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME' now contains a key pair and a" echo "self-signed certificate. Again, this keystore can only be used for one broker or" echo "one logical client. Other brokers or clients need to generate their own keystores." echo echo "Fetching the certificate from the trust store and storing in $CA_CERT_FILE." echo keytool -keystore $trust_store_file -export -alias CARoot -rfc -file $CA_CERT_FILE -keypass $PASS -storepass $PASS echo echo "Now a certificate signing request will be made to the keystore." echo keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost \ -certreq -file $KEYSTORE_SIGN_REQUEST -keypass $PASS -storepass $PASS echo echo "Now the trust store's private key (CA) will sign the keystore's certificate." echo openssl x509 -req -CA $CA_CERT_FILE -CAkey $trust_store_private_key_file \ -in $KEYSTORE_SIGN_REQUEST -out $KEYSTORE_SIGNED_CERT \ -days $VALIDITY_IN_DAYS -CAcreateserial \ -extensions kafka -extfile san.cnf # creates $KEYSTORE_SIGN_REQUEST_SRL which is never used or needed. echo echo "Now the CA will be imported into the keystore." echo keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias CARoot \ -import -file $CA_CERT_FILE -keypass $PASS -storepass $PASS -noprompt rm $CA_CERT_FILE # delete the trust store cert because it's stored in the trust store. echo echo "Now the keystore's signed certificate will be imported back into the keystore." echo keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost -import \ -file $KEYSTORE_SIGNED_CERT -keypass $PASS -storepass $PASS echo echo "All done!" echo echo "Deleting intermediate files. They are:" echo " - '$KEYSTORE_SIGN_REQUEST_SRL': CA serial number" echo " - '$KEYSTORE_SIGN_REQUEST': the keystore's certificate signing request" echo " (that was fulfilled)" echo " - '$KEYSTORE_SIGNED_CERT': the keystore's certificate, signed by the CA, and stored back" echo " into the keystore" rm $KEYSTORE_SIGN_REQUEST_SRL rm $KEYSTORE_SIGN_REQUEST rm $KEYSTORE_SIGNED_CERT ================================================ FILE: documentation/compose/ssl/san.cnf ================================================ [kafka] subjectAltName = DNS:kafka0,DNS:schemaregistry0,DNS:kafka-connect0,DNS:ksqldb0 ================================================ FILE: documentation/compose/traefik/kafkaui.yaml ================================================ http: routers: kafkaui: rule: "PathPrefix(`/kafka-ui/`)" entrypoints: web service: kafkaui services: kafkaui: loadBalancer: servers: - url: http://kafka-ui:8080 ================================================ FILE: documentation/compose/traefik-proxy.yaml ================================================ --- version: '3.8' services: traefik: restart: always image: traefik:v2.4 container_name: traefik command: - --api.insecure=true - --providers.file.directory=/etc/traefik - --providers.file.watch=true - --entrypoints.web.address=:80 - --log.level=debug ports: - 80:80 volumes: - ./traefik:/etc/traefik kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:latest ports: - 8082:8080 environment: KAFKA_CLUSTERS_0_NAME: local KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092 SERVER_SERVLET_CONTEXT_PATH: /kafka-ui ================================================ FILE: etc/checkstyle/apache-header.txt ================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: etc/checkstyle/checkstyle-e2e.xml ================================================ ================================================ FILE: etc/checkstyle/checkstyle.xml ================================================ ================================================ FILE: kafka-ui-api/Dockerfile ================================================ #FROM azul/zulu-openjdk-alpine:17-jre-headless FROM azul/zulu-openjdk-alpine@sha256:a36679ac0d28cb835e2a8c00e1e0d95509c6c51c5081c7782b85edb1f37a771a RUN apk add --no-cache \ # snappy codec gcompat \ # configuring timezones tzdata RUN addgroup -S kafkaui && adduser -S kafkaui -G kafkaui # creating folder for dynamic config usage (certificates uploads, etc) RUN mkdir /etc/kafkaui/ RUN chown kafkaui /etc/kafkaui USER kafkaui ARG JAR_FILE COPY "/target/${JAR_FILE}" "/kafka-ui-api.jar" ENV JAVA_OPTS= EXPOSE 8080 # see JmxSslSocketFactory docs to understand why add-opens is needed CMD java --add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED $JAVA_OPTS -jar kafka-ui-api.jar ================================================ FILE: kafka-ui-api/pom.xml ================================================ kafka-ui com.provectus 0.0.1-SNAPSHOT 4.0.0 kafka-ui-api 0.8.10 jacoco reuseReports ${project.basedir}/target/jacoco.exec ${project.basedir}/target/site/jacoco/jacoco.xml java org.springframework.boot spring-boot-starter-webflux org.springframework.boot spring-boot-starter-security org.springframework.boot spring-boot-actuator org.springframework.boot spring-boot-starter-oauth2-client com.provectus kafka-ui-contract ${project.version} com.provectus kafka-ui-serde-api ${kafka-ui-serde-api.version} org.apache.kafka kafka-clients ${kafka-clients.version} org.apache.commons commons-lang3 3.12.0 org.projectlombok lombok provided org.mapstruct mapstruct ${org.mapstruct.version} io.confluent kafka-schema-registry-client ${confluent.version} io.confluent kafka-avro-serializer ${confluent.version} io.confluent kafka-json-schema-serializer ${confluent.version} commons-collections commons-collections io.confluent kafka-protobuf-serializer ${confluent.version} software.amazon.msk aws-msk-iam-auth 1.1.7 org.apache.avro avro ${avro.version} org.springframework.boot spring-boot-starter-logging io.projectreactor.addons reactor-extra org.json json ${org.json.version} io.micrometer micrometer-registry-prometheus runtime org.springframework.boot spring-boot-starter-test test io.projectreactor reactor-test test org.apache.commons commons-pool2 ${apache.commons.version} org.apache.commons commons-collections4 4.4 org.testcontainers testcontainers test org.testcontainers kafka test org.testcontainers junit-jupiter test org.junit.jupiter junit-jupiter-engine test org.mockito mockito-core ${mockito.version} test org.mockito mockito-junit-jupiter ${mockito.version} test net.bytebuddy byte-buddy ${byte-buddy.version} test org.assertj assertj-core ${assertj.version} test com.github.java-json-tools json-schema-validator 2.2.14 test com.squareup.okhttp3 mockwebserver ${okhttp3.mockwebserver.version} test com.squareup.okhttp3 okhttp ${okhttp3.mockwebserver.version} test org.springframework.boot spring-boot-starter-actuator org.antlr antlr4-runtime ${antlr4-maven-plugin.version} org.opendatadiscovery oddrn-generator-java ${odd-oddrn-generator.version} org.opendatadiscovery ingestion-contract-client org.springframework.boot spring-boot-starter-webflux io.projectreactor reactor-core io.projectreactor.ipc reactor-netty ${odd-oddrn-client.version} org.springframework.security spring-security-ldap org.codehaus.groovy groovy-jsr223 ${groovy.version} org.codehaus.groovy groovy-json ${groovy.version} org.apache.datasketches datasketches-java ${datasketches-java.version} org.springframework.boot spring-boot-devtools true org.springframework.boot spring-boot-maven-plugin ${spring-boot.version} repackage build-info org.apache.maven.plugins maven-compiler-plugin org.mapstruct mapstruct-processor ${org.mapstruct.version} org.projectlombok lombok ${org.projectlombok.version} org.projectlombok lombok-mapstruct-binding 0.2.0 org.springframework.boot spring-boot-configuration-processor ${spring-boot.version} org.apache.maven.plugins maven-surefire-plugin @{argLine} --illegal-access=permit org.apache.maven.plugins maven-checkstyle-plugin 3.3.0 com.puppycrawl.tools checkstyle 10.3.1 checkstyle validate check warning true true true file:${basedir}/../etc/checkstyle/checkstyle.xml file:${basedir}/../etc/checkstyle/apache-header.txt org.antlr antlr4-maven-plugin ${antlr4-maven-plugin.version} false generate-sources antlr4 org.jacoco jacoco-maven-plugin ${jacoco.version} prepare-agent prepare-agent report report XML prod pl.project13.maven git-commit-id-plugin 4.9.10 get-the-git-infos revision initialize true ${project.build.outputDirectory}/git.properties ^git.build.(time|version)$ ^git.commit.id.(abbrev|full)$ full maven-resources-plugin copy-resources process-classes copy-resources ${basedir}/target/classes/static ../kafka-ui-react-app/build com.github.eirslett frontend-maven-plugin ${frontend-maven-plugin.version} ${skipUIBuild} ../kafka-ui-react-app ${project.version} ${git.commit.id.abbrev} install node and pnpm install-node-and-pnpm ${node.version} ${pnpm.version} pnpm install pnpm install pnpm build pnpm build io.fabric8 docker-maven-plugin ${fabric8-maven-plugin.version} true provectuslabs/kafka-ui:${git.revision} ${project.basedir} ${project.build.finalName}.jar default package build ================================================ FILE: kafka-ui-api/src/main/antlr4/ksql/KsqlGrammar.g4 ================================================ grammar KsqlGrammar; tokens { DELIMITER } @lexer::members { public static final int COMMENTS = 2; public static final int WHITESPACE = 3; public static final int DIRECTIVES = 4; } statements : (singleStatement)* EOF ; testStatement : (singleStatement | assertStatement ';' | runScript ';') EOF? ; singleStatement : statement ';' ; singleExpression : expression EOF ; statement : query #queryStatement | (LIST | SHOW) PROPERTIES #listProperties | (LIST | SHOW) ALL? TOPICS EXTENDED? #listTopics | (LIST | SHOW) STREAMS EXTENDED? #listStreams | (LIST | SHOW) TABLES EXTENDED? #listTables | (LIST | SHOW) FUNCTIONS #listFunctions | (LIST | SHOW) (SOURCE | SINK)? CONNECTORS #listConnectors | (LIST | SHOW) CONNECTOR PLUGINS #listConnectorPlugins | (LIST | SHOW) TYPES #listTypes | (LIST | SHOW) VARIABLES #listVariables | DESCRIBE sourceName EXTENDED? #showColumns | DESCRIBE STREAMS EXTENDED? #describeStreams | DESCRIBE FUNCTION identifier #describeFunction | DESCRIBE CONNECTOR identifier #describeConnector | PRINT (identifier| STRING) printClause #printTopic | (LIST | SHOW) QUERIES EXTENDED? #listQueries | TERMINATE identifier #terminateQuery | TERMINATE ALL #terminateQuery | SET STRING EQ STRING #setProperty | UNSET STRING #unsetProperty | DEFINE variableName EQ variableValue #defineVariable | UNDEFINE variableName #undefineVariable | CREATE (OR REPLACE)? (SOURCE)? STREAM (IF NOT EXISTS)? sourceName (tableElements)? (WITH tableProperties)? #createStream | CREATE (OR REPLACE)? STREAM (IF NOT EXISTS)? sourceName (WITH tableProperties)? AS query #createStreamAs | CREATE (OR REPLACE)? (SOURCE)? TABLE (IF NOT EXISTS)? sourceName (tableElements)? (WITH tableProperties)? #createTable | CREATE (OR REPLACE)? TABLE (IF NOT EXISTS)? sourceName (WITH tableProperties)? AS query #createTableAs | CREATE (SINK | SOURCE) CONNECTOR (IF NOT EXISTS)? identifier WITH tableProperties #createConnector | INSERT INTO sourceName (WITH tableProperties)? query #insertInto | INSERT INTO sourceName (columns)? VALUES values #insertValues | DROP STREAM (IF EXISTS)? sourceName (DELETE TOPIC)? #dropStream | DROP TABLE (IF EXISTS)? sourceName (DELETE TOPIC)? #dropTable | DROP CONNECTOR (IF EXISTS)? identifier #dropConnector | EXPLAIN (statement | identifier) #explain | CREATE TYPE (IF NOT EXISTS)? identifier AS type #registerType | DROP TYPE (IF EXISTS)? identifier #dropType | ALTER (STREAM | TABLE) sourceName alterOption (',' alterOption)* #alterSource ; assertStatement : ASSERT VALUES sourceName (columns)? VALUES values #assertValues | ASSERT NULL VALUES sourceName (columns)? KEY values #assertTombstone | ASSERT STREAM sourceName (tableElements)? (WITH tableProperties)? #assertStream | ASSERT TABLE sourceName (tableElements)? (WITH tableProperties)? #assertTable ; runScript : RUN SCRIPT STRING ; query : SELECT selectItem (',' selectItem)* FROM from=relation (WINDOW windowExpression)? (WHERE where=booleanExpression)? (GROUP BY groupBy)? (PARTITION BY partitionBy)? (HAVING having=booleanExpression)? (EMIT resultMaterialization)? limitClause? ; resultMaterialization : CHANGES | FINAL ; alterOption : ADD (COLUMN)? identifier type ; tableElements : '(' tableElement (',' tableElement)* ')' ; tableElement : identifier type columnConstraints? ; columnConstraints : ((PRIMARY)? KEY) | HEADERS | HEADER '(' STRING ')' ; tableProperties : '(' tableProperty (',' tableProperty)* ')' ; tableProperty : (identifier | STRING) EQ literal ; printClause : (FROM BEGINNING)? intervalClause? limitClause? ; intervalClause : (INTERVAL | SAMPLE) number ; limitClause : LIMIT number ; retentionClause : RETENTION number windowUnit ; gracePeriodClause : GRACE PERIOD number windowUnit ; windowExpression : (IDENTIFIER)? ( tumblingWindowExpression | hoppingWindowExpression | sessionWindowExpression ) ; tumblingWindowExpression : TUMBLING '(' SIZE number windowUnit (',' retentionClause)? (',' gracePeriodClause)?')' ; hoppingWindowExpression : HOPPING '(' SIZE number windowUnit ',' ADVANCE BY number windowUnit (',' retentionClause)? (',' gracePeriodClause)?')' ; sessionWindowExpression : SESSION '(' number windowUnit (',' retentionClause)? (',' gracePeriodClause)?')' ; windowUnit : DAY | HOUR | MINUTE | SECOND | MILLISECOND | DAYS | HOURS | MINUTES | SECONDS | MILLISECONDS ; groupBy : valueExpression (',' valueExpression)* | '(' (valueExpression (',' valueExpression)*)? ')' ; partitionBy : valueExpression (',' valueExpression)* | '(' (valueExpression (',' valueExpression)*)? ')' ; values : '(' (valueExpression (',' valueExpression)*)? ')' ; selectItem : expression (AS? identifier)? #selectSingle | identifier '.' ASTERISK #selectAll | ASTERISK #selectAll ; relation : left=aliasedRelation joinedSource+ #joinRelation | aliasedRelation #relationDefault ; joinedSource : joinType JOIN aliasedRelation joinWindow? joinCriteria ; joinType : INNER? #innerJoin | FULL OUTER? #outerJoin | LEFT OUTER? #leftJoin ; joinWindow : WITHIN withinExpression ; withinExpression : '(' joinWindowSize ',' joinWindowSize ')' (gracePeriodClause)? # joinWindowWithBeforeAndAfter | joinWindowSize (gracePeriodClause)? # singleJoinWindow ; joinWindowSize : number windowUnit ; joinCriteria : ON booleanExpression ; aliasedRelation : relationPrimary (AS? sourceName)? ; columns : '(' identifier (',' identifier)* ')' ; relationPrimary : sourceName #tableName ; expression : booleanExpression ; booleanExpression : predicated #booleanDefault | NOT booleanExpression #logicalNot | left=booleanExpression operator=AND right=booleanExpression #logicalBinary | left=booleanExpression operator=OR right=booleanExpression #logicalBinary ; predicated : valueExpression predicate[$valueExpression.ctx]? ; predicate[ParserRuleContext value] : comparisonOperator right=valueExpression #comparison | NOT? BETWEEN lower=valueExpression AND upper=valueExpression #between | NOT? IN '(' expression (',' expression)* ')' #inList | NOT? LIKE pattern=valueExpression (ESCAPE escape=STRING)? #like | IS NOT? NULL #nullPredicate | IS NOT? DISTINCT FROM right=valueExpression #distinctFrom ; valueExpression : primaryExpression #valueExpressionDefault | valueExpression AT timeZoneSpecifier #atTimeZone | operator=(MINUS | PLUS) valueExpression #arithmeticUnary | left=valueExpression operator=(ASTERISK | SLASH | PERCENT) right=valueExpression #arithmeticBinary | left=valueExpression operator=(PLUS | MINUS) right=valueExpression #arithmeticBinary | left=valueExpression CONCAT right=valueExpression #concatenation ; primaryExpression : literal #literalExpression | identifier STRING #typeConstructor | CASE valueExpression whenClause+ (ELSE elseExpression=expression)? END #simpleCase | CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase | CAST '(' expression AS type ')' #cast | ARRAY '[' (expression (',' expression)*)? ']' #arrayConstructor | MAP '(' (expression ASSIGN expression (',' expression ASSIGN expression)*)? ')' #mapConstructor | STRUCT '(' (identifier ASSIGN expression (',' identifier ASSIGN expression)*)? ')' #structConstructor | identifier '(' ASTERISK ')' #functionCall | identifier '(' (functionArgument (',' functionArgument)* (',' lambdaFunction)*)? ')' #functionCall | value=primaryExpression '[' index=valueExpression ']' #subscript | identifier #columnReference | identifier '.' identifier #qualifiedColumnReference | base=primaryExpression STRUCT_FIELD_REF fieldName=identifier #dereference | '(' expression ')' #parenthesizedExpression ; functionArgument : expression | windowUnit ; timeZoneSpecifier : TIME ZONE STRING #timeZoneString ; comparisonOperator : EQ | NEQ | LT | LTE | GT | GTE ; booleanValue : TRUE | FALSE ; type : type ARRAY | ARRAY '<' type '>' | MAP '<' type ',' type '>' | STRUCT '<' (identifier type (',' identifier type)*)? '>' | DECIMAL '(' number ',' number ')' | baseType ('(' typeParameter (',' typeParameter)* ')')? ; typeParameter : INTEGER_VALUE | 'STRING' ; baseType : identifier ; whenClause : WHEN condition=expression THEN result=expression ; identifier : VARIABLE #variableIdentifier | IDENTIFIER #unquotedIdentifier | QUOTED_IDENTIFIER #quotedIdentifierAlternative | nonReserved #unquotedIdentifier | BACKQUOTED_IDENTIFIER #backQuotedIdentifier | DIGIT_IDENTIFIER #digitIdentifier ; lambdaFunction : identifier '=>' expression #lambda | '(' identifier (',' identifier)* ')' '=>' expression #lambda ; variableName : IDENTIFIER ; variableValue : STRING ; sourceName : identifier ; number : MINUS? DECIMAL_VALUE #decimalLiteral | MINUS? FLOATING_POINT_VALUE #floatLiteral | MINUS? INTEGER_VALUE #integerLiteral ; literal : NULL #nullLiteral | number #numericLiteral | booleanValue #booleanLiteral | STRING #stringLiteral | VARIABLE #variableLiteral ; nonReserved : SHOW | TABLES | COLUMNS | COLUMN | PARTITIONS | FUNCTIONS | FUNCTION | SESSION | STRUCT | MAP | ARRAY | PARTITION | INTEGER | DATE | TIME | TIMESTAMP | INTERVAL | ZONE | 'STRING' | YEAR | MONTH | DAY | HOUR | MINUTE | SECOND | EXPLAIN | ANALYZE | TYPE | TYPES | SET | RESET | IF | SOURCE | SINK | PRIMARY | KEY | EMIT | CHANGES | FINAL | ESCAPE | REPLACE | ASSERT | ALTER | ADD ; EMIT: 'EMIT'; CHANGES: 'CHANGES'; FINAL: 'FINAL'; SELECT: 'SELECT'; FROM: 'FROM'; AS: 'AS'; ALL: 'ALL'; DISTINCT: 'DISTINCT'; WHERE: 'WHERE'; WITHIN: 'WITHIN'; WINDOW: 'WINDOW'; GROUP: 'GROUP'; BY: 'BY'; HAVING: 'HAVING'; LIMIT: 'LIMIT'; AT: 'AT'; OR: 'OR'; AND: 'AND'; IN: 'IN'; NOT: 'NOT'; EXISTS: 'EXISTS'; BETWEEN: 'BETWEEN'; LIKE: 'LIKE'; ESCAPE: 'ESCAPE'; IS: 'IS'; NULL: 'NULL'; TRUE: 'TRUE'; FALSE: 'FALSE'; INTEGER: 'INTEGER'; DATE: 'DATE'; TIME: 'TIME'; TIMESTAMP: 'TIMESTAMP'; INTERVAL: 'INTERVAL'; YEAR: 'YEAR'; MONTH: 'MONTH'; DAY: 'DAY'; HOUR: 'HOUR'; MINUTE: 'MINUTE'; SECOND: 'SECOND'; MILLISECOND: 'MILLISECOND'; YEARS: 'YEARS'; MONTHS: 'MONTHS'; DAYS: 'DAYS'; HOURS: 'HOURS'; MINUTES: 'MINUTES'; SECONDS: 'SECONDS'; MILLISECONDS: 'MILLISECONDS'; ZONE: 'ZONE'; TUMBLING: 'TUMBLING'; HOPPING: 'HOPPING'; SIZE: 'SIZE'; ADVANCE: 'ADVANCE'; RETENTION: 'RETENTION'; GRACE: 'GRACE'; PERIOD: 'PERIOD'; CASE: 'CASE'; WHEN: 'WHEN'; THEN: 'THEN'; ELSE: 'ELSE'; END: 'END'; JOIN: 'JOIN'; FULL: 'FULL'; OUTER: 'OUTER'; INNER: 'INNER'; LEFT: 'LEFT'; RIGHT: 'RIGHT'; ON: 'ON'; PARTITION: 'PARTITION'; STRUCT: 'STRUCT'; WITH: 'WITH'; VALUES: 'VALUES'; CREATE: 'CREATE'; TABLE: 'TABLE'; TOPIC: 'TOPIC'; STREAM: 'STREAM'; STREAMS: 'STREAMS'; INSERT: 'INSERT'; DELETE: 'DELETE'; INTO: 'INTO'; DESCRIBE: 'DESCRIBE'; EXTENDED: 'EXTENDED'; PRINT: 'PRINT'; EXPLAIN: 'EXPLAIN'; ANALYZE: 'ANALYZE'; TYPE: 'TYPE'; TYPES: 'TYPES'; CAST: 'CAST'; SHOW: 'SHOW'; LIST: 'LIST'; TABLES: 'TABLES'; TOPICS: 'TOPICS'; QUERY: 'QUERY'; QUERIES: 'QUERIES'; TERMINATE: 'TERMINATE'; LOAD: 'LOAD'; COLUMNS: 'COLUMNS'; COLUMN: 'COLUMN'; PARTITIONS: 'PARTITIONS'; FUNCTIONS: 'FUNCTIONS'; FUNCTION: 'FUNCTION'; DROP: 'DROP'; TO: 'TO'; RENAME: 'RENAME'; ARRAY: 'ARRAY'; MAP: 'MAP'; SET: 'SET'; DEFINE: 'DEFINE'; UNDEFINE: 'UNDEFINE'; RESET: 'RESET'; SESSION: 'SESSION'; SAMPLE: 'SAMPLE'; EXPORT: 'EXPORT'; CATALOG: 'CATALOG'; PROPERTIES: 'PROPERTIES'; BEGINNING: 'BEGINNING'; UNSET: 'UNSET'; RUN: 'RUN'; SCRIPT: 'SCRIPT'; DECIMAL: 'DECIMAL'; KEY: 'KEY'; CONNECTOR: 'CONNECTOR'; CONNECTORS: 'CONNECTORS'; SINK: 'SINK'; SOURCE: 'SOURCE'; NAMESPACE: 'NAMESPACE'; MATERIALIZED: 'MATERIALIZED'; VIEW: 'VIEW'; PRIMARY: 'PRIMARY'; REPLACE: 'REPLACE'; ASSERT: 'ASSERT'; ADD: 'ADD'; ALTER: 'ALTER'; VARIABLES: 'VARIABLES'; PLUGINS: 'PLUGINS'; HEADERS: 'HEADERS'; HEADER: 'HEADER'; IF: 'IF'; EQ : '='; NEQ : '<>' | '!='; LT : '<'; LTE : '<='; GT : '>'; GTE : '>='; PLUS: '+'; MINUS: '-'; ASTERISK: '*'; SLASH: '/'; PERCENT: '%'; CONCAT: '||'; ASSIGN: ':='; STRUCT_FIELD_REF: '->'; LAMBDA_EXPRESSION: '=>'; STRING : '\'' ( ~'\'' | '\'\'' )* '\'' ; INTEGER_VALUE : DIGIT+ ; DECIMAL_VALUE : DIGIT+ '.' DIGIT* | '.' DIGIT+ ; FLOATING_POINT_VALUE : DIGIT+ ('.' DIGIT*)? EXPONENT | '.' DIGIT+ EXPONENT ; IDENTIFIER : (LETTER | '_') (LETTER | DIGIT | '_' | '@' )* ; DIGIT_IDENTIFIER : DIGIT (LETTER | DIGIT | '_' | '@' )+ ; QUOTED_IDENTIFIER : '"' ( ~'"' | '""' )* '"' ; BACKQUOTED_IDENTIFIER : '`' ( ~'`' | '``' )* '`' ; VARIABLE : '${' IDENTIFIER '}' ; fragment EXPONENT : 'E' [+-]? DIGIT+ ; fragment DIGIT : [0-9] ; fragment LETTER : [A-Z] ; SIMPLE_COMMENT : '--' ~'@' ~[\r\n]* '\r'? '\n'? -> channel(2) // channel(COMMENTS) ; DIRECTIVE_COMMENT : '--@' ~[\r\n]* '\r'? '\n'? -> channel(4) // channel(DIRECTIVES) ; BRACKETED_COMMENT : '/*' .*? '*/' -> channel(2) // channel(COMMENTS) ; WS : [ \r\n\t]+ -> channel(3) // channel(WHITESPACE) ; // Catch-all for anything we can't recognize. // We use this to be able to ignore and recover all the text // when splitting statements with DelimiterLexer UNRECOGNIZED : . ; ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/KafkaUiApplication.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.util.DynamicConfigOperations; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication(exclude = LdapAutoConfiguration.class) @EnableScheduling @EnableAsync public class KafkaUiApplication { public static void main(String[] args) { startApplication(args); } public static ConfigurableApplicationContext startApplication(String[] args) { return new SpringApplicationBuilder(KafkaUiApplication.class) .initializers(DynamicConfigOperations.dynamicConfigPropertiesInitializer()) .build() .run(args); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/RetryingKafkaConnectClient.java ================================================ package com.provectus.kafka.ui.client; import static com.provectus.kafka.ui.config.ClustersProperties.ConnectCluster; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.connect.ApiClient; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.connect.model.Connector; import com.provectus.kafka.ui.connect.model.ConnectorPlugin; import com.provectus.kafka.ui.connect.model.ConnectorPluginConfigValidationResponse; import com.provectus.kafka.ui.connect.model.ConnectorStatus; import com.provectus.kafka.ui.connect.model.ConnectorTask; import com.provectus.kafka.ui.connect.model.ConnectorTopics; import com.provectus.kafka.ui.connect.model.NewConnector; import com.provectus.kafka.ui.connect.model.TaskStatus; import com.provectus.kafka.ui.exception.KafkaConnectConflictReponseException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.util.WebClientConfigurator; import java.time.Duration; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.util.unit.DataSize; import org.springframework.web.client.RestClientException; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.retry.Retry; @Slf4j public class RetryingKafkaConnectClient extends KafkaConnectClientApi { private static final int MAX_RETRIES = 5; private static final Duration RETRIES_DELAY = Duration.ofMillis(200); public RetryingKafkaConnectClient(ConnectCluster config, @Nullable ClustersProperties.TruststoreConfig truststoreConfig, DataSize maxBuffSize) { super(new RetryingApiClient(config, truststoreConfig, maxBuffSize)); } private static Retry conflictCodeRetry() { return Retry .fixedDelay(MAX_RETRIES, RETRIES_DELAY) .filter(e -> e instanceof WebClientResponseException.Conflict) .onRetryExhaustedThrow((spec, signal) -> new KafkaConnectConflictReponseException( (WebClientResponseException.Conflict) signal.failure())); } private static Mono withRetryOnConflict(Mono publisher) { return publisher.retryWhen(conflictCodeRetry()); } private static Flux withRetryOnConflict(Flux publisher) { return publisher.retryWhen(conflictCodeRetry()); } private static Mono withBadRequestErrorHandling(Mono publisher) { return publisher .onErrorResume(WebClientResponseException.BadRequest.class, e -> Mono.error(new ValidationException("Invalid configuration"))) .onErrorResume(WebClientResponseException.InternalServerError.class, e -> Mono.error(new ValidationException("Invalid configuration"))); } @Override public Mono createConnector(NewConnector newConnector) throws RestClientException { return withBadRequestErrorHandling( super.createConnector(newConnector) ); } @Override public Mono setConnectorConfig(String connectorName, Map requestBody) throws RestClientException { return withBadRequestErrorHandling( super.setConnectorConfig(connectorName, requestBody) ); } @Override public Mono> createConnectorWithHttpInfo(NewConnector newConnector) throws WebClientResponseException { return withRetryOnConflict(super.createConnectorWithHttpInfo(newConnector)); } @Override public Mono deleteConnector(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.deleteConnector(connectorName)); } @Override public Mono> deleteConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.deleteConnectorWithHttpInfo(connectorName)); } @Override public Mono getConnector(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnector(connectorName)); } @Override public Mono> getConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorWithHttpInfo(connectorName)); } @Override public Mono> getConnectorConfig(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorConfig(connectorName)); } @Override public Mono>> getConnectorConfigWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorConfigWithHttpInfo(connectorName)); } @Override public Flux getConnectorPlugins() throws WebClientResponseException { return withRetryOnConflict(super.getConnectorPlugins()); } @Override public Mono>> getConnectorPluginsWithHttpInfo() throws WebClientResponseException { return withRetryOnConflict(super.getConnectorPluginsWithHttpInfo()); } @Override public Mono getConnectorStatus(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorStatus(connectorName)); } @Override public Mono> getConnectorStatusWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorStatusWithHttpInfo(connectorName)); } @Override public Mono getConnectorTaskStatus(String connectorName, Integer taskId) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTaskStatus(connectorName, taskId)); } @Override public Mono> getConnectorTaskStatusWithHttpInfo(String connectorName, Integer taskId) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTaskStatusWithHttpInfo(connectorName, taskId)); } @Override public Flux getConnectorTasks(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTasks(connectorName)); } @Override public Mono>> getConnectorTasksWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTasksWithHttpInfo(connectorName)); } @Override public Mono> getConnectorTopics(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTopics(connectorName)); } @Override public Mono>> getConnectorTopicsWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorTopicsWithHttpInfo(connectorName)); } @Override public Flux getConnectors(String search) throws WebClientResponseException { return withRetryOnConflict(super.getConnectors(search)); } @Override public Mono>> getConnectorsWithHttpInfo(String search) throws WebClientResponseException { return withRetryOnConflict(super.getConnectorsWithHttpInfo(search)); } @Override public Mono pauseConnector(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.pauseConnector(connectorName)); } @Override public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.pauseConnectorWithHttpInfo(connectorName)); } @Override public Mono restartConnector(String connectorName, Boolean includeTasks, Boolean onlyFailed) throws WebClientResponseException { return withRetryOnConflict(super.restartConnector(connectorName, includeTasks, onlyFailed)); } @Override public Mono> restartConnectorWithHttpInfo(String connectorName, Boolean includeTasks, Boolean onlyFailed) throws WebClientResponseException { return withRetryOnConflict(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); } @Override public Mono restartConnectorTask(String connectorName, Integer taskId) throws WebClientResponseException { return withRetryOnConflict(super.restartConnectorTask(connectorName, taskId)); } @Override public Mono> restartConnectorTaskWithHttpInfo(String connectorName, Integer taskId) throws WebClientResponseException { return withRetryOnConflict(super.restartConnectorTaskWithHttpInfo(connectorName, taskId)); } @Override public Mono resumeConnector(String connectorName) throws WebClientResponseException { return super.resumeConnector(connectorName); } @Override public Mono> resumeConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { return withRetryOnConflict(super.resumeConnectorWithHttpInfo(connectorName)); } @Override public Mono> setConnectorConfigWithHttpInfo(String connectorName, Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); } @Override public Mono validateConnectorPluginConfig(String pluginName, Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.validateConnectorPluginConfig(pluginName, requestBody)); } @Override public Mono> validateConnectorPluginConfigWithHttpInfo( String pluginName, Map requestBody) throws WebClientResponseException { return withRetryOnConflict(super.validateConnectorPluginConfigWithHttpInfo(pluginName, requestBody)); } private static class RetryingApiClient extends ApiClient { public RetryingApiClient(ConnectCluster config, ClustersProperties.TruststoreConfig truststoreConfig, DataSize maxBuffSize) { super(buildWebClient(maxBuffSize, config, truststoreConfig), null, null); setBasePath(config.getAddress()); setUsername(config.getUsername()); setPassword(config.getPassword()); } public static WebClient buildWebClient(DataSize maxBuffSize, ConnectCluster config, ClustersProperties.TruststoreConfig truststoreConfig) { return new WebClientConfigurator() .configureSsl( truststoreConfig, new ClustersProperties.KeystoreConfig( config.getKeystoreLocation(), config.getKeystorePassword() ) ) .configureBasicAuth( config.getUsername(), config.getPassword() ) .configureBufferSize(maxBuffSize) .build(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java ================================================ package com.provectus.kafka.ui.config; import com.provectus.kafka.ui.model.MetricsConfig; import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.util.StringUtils; @Configuration @ConfigurationProperties("kafka") @Data public class ClustersProperties { List clusters = new ArrayList<>(); String internalTopicPrefix; Integer adminClientTimeout; PollingProperties polling = new PollingProperties(); @Data public static class Cluster { String name; String bootstrapServers; String schemaRegistry; SchemaRegistryAuth schemaRegistryAuth; KeystoreConfig schemaRegistrySsl; String ksqldbServer; KsqldbServerAuth ksqldbServerAuth; KeystoreConfig ksqldbServerSsl; List kafkaConnect; MetricsConfigData metrics; Map properties; boolean readOnly = false; List serde; String defaultKeySerde; String defaultValueSerde; List masking; Long pollingThrottleRate; TruststoreConfig ssl; AuditProperties audit; } @Data public static class PollingProperties { Integer pollTimeoutMs; Integer maxPageSize; Integer defaultPageSize; } @Data @ToString(exclude = "password") public static class MetricsConfigData { String type; Integer port; Boolean ssl; String username; String password; String keystoreLocation; String keystorePassword; } @Data @NoArgsConstructor @AllArgsConstructor @Builder(toBuilder = true) @ToString(exclude = {"password", "keystorePassword"}) public static class ConnectCluster { String name; String address; String username; String password; String keystoreLocation; String keystorePassword; } @Data @ToString(exclude = {"password"}) public static class SchemaRegistryAuth { String username; String password; } @Data @ToString(exclude = {"truststorePassword"}) public static class TruststoreConfig { String truststoreLocation; String truststorePassword; } @Data public static class SerdeConfig { String name; String className; String filePath; Map properties; String topicKeysPattern; String topicValuesPattern; } @Data @ToString(exclude = "password") public static class KsqldbServerAuth { String username; String password; } @Data @NoArgsConstructor @AllArgsConstructor @ToString(exclude = {"keystorePassword"}) public static class KeystoreConfig { String keystoreLocation; String keystorePassword; } @Data public static class Masking { Type type; List fields; String fieldsNamePattern; List maskingCharsReplacement; //used when type=MASK String replacement; //used when type=REPLACE String topicKeysPattern; String topicValuesPattern; public enum Type { REMOVE, MASK, REPLACE } } @Data @NoArgsConstructor @AllArgsConstructor public static class AuditProperties { String topic; Integer auditTopicsPartitions; Boolean topicAuditEnabled; Boolean consoleAuditEnabled; LogLevel level; Map auditTopicProperties; public enum LogLevel { ALL, ALTER_ONLY //default } } @PostConstruct public void validateAndSetDefaults() { if (clusters != null) { validateClusterNames(); flattenClusterProperties(); setMetricsDefaults(); } } private void setMetricsDefaults() { for (Cluster cluster : clusters) { if (cluster.getMetrics() != null && !StringUtils.hasText(cluster.getMetrics().getType())) { cluster.getMetrics().setType(MetricsConfig.JMX_METRICS_TYPE); } } } private void flattenClusterProperties() { for (Cluster cluster : clusters) { cluster.setProperties(flattenClusterProperties(null, cluster.getProperties())); } } private Map flattenClusterProperties(@Nullable String prefix, @Nullable Map propertiesMap) { Map flattened = new HashMap<>(); if (propertiesMap != null) { propertiesMap.forEach((k, v) -> { String key = prefix == null ? k : prefix + "." + k; if (v instanceof Map) { flattened.putAll(flattenClusterProperties(key, (Map) v)); } else { flattened.put(key, v); } }); } return flattened; } private void validateClusterNames() { // if only one cluster provided it is ok not to set name if (clusters.size() == 1 && !StringUtils.hasText(clusters.get(0).getName())) { clusters.get(0).setName("Default"); return; } Set clusterNames = new HashSet<>(); for (Cluster clusterProperties : clusters) { if (!StringUtils.hasText(clusterProperties.getName())) { throw new IllegalStateException( "Application config isn't valid. " + "Cluster names should be provided in case of multiple clusters present"); } if (!clusterNames.add(clusterProperties.getName())) { throw new IllegalStateException( "Application config isn't valid. Two clusters can't have the same name"); } } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/Config.java ================================================ package com.provectus.kafka.ui.config; import java.util.Collections; import java.util.Map; import lombok.AllArgsConstructor; import org.openapitools.jackson.nullable.JsonNullableModule; import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.web.ServerProperties; import org.springframework.boot.autoconfigure.web.reactive.WebFluxProperties; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.server.reactive.ContextPathCompositeHandler; import org.springframework.http.server.reactive.HttpHandler; import org.springframework.jmx.export.MBeanExporter; import org.springframework.util.StringUtils; import org.springframework.web.server.adapter.WebHttpHandlerBuilder; @Configuration @AllArgsConstructor public class Config { private final ApplicationContext applicationContext; private final ServerProperties serverProperties; @Bean public HttpHandler httpHandler(ObjectProvider propsProvider) { final String basePath = serverProperties.getServlet().getContextPath(); HttpHandler httpHandler = WebHttpHandlerBuilder .applicationContext(this.applicationContext).build(); if (StringUtils.hasText(basePath)) { Map handlersMap = Collections.singletonMap(basePath, httpHandler); return new ContextPathCompositeHandler(handlersMap); } return httpHandler; } @Bean public MBeanExporter exporter() { final var exporter = new MBeanExporter(); exporter.setAutodetect(true); exporter.setExcludedBeans("pool"); return exporter; } @Bean // will be used by webflux json mapping public JsonNullableModule jsonNullableModule() { return new JsonNullableModule(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/CorsGlobalConfiguration.java ================================================ package com.provectus.kafka.ui.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilter; import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; @Configuration public class CorsGlobalConfiguration { @Bean public WebFilter corsFilter() { return (final ServerWebExchange ctx, final WebFilterChain chain) -> { final ServerHttpRequest request = ctx.getRequest(); final ServerHttpResponse response = ctx.getResponse(); final HttpHeaders headers = response.getHeaders(); headers.add("Access-Control-Allow-Origin", "*"); headers.add("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, OPTIONS"); headers.add("Access-Control-Max-Age", "3600"); headers.add("Access-Control-Allow-Headers", "Content-Type"); if (request.getMethod() == HttpMethod.OPTIONS) { response.setStatusCode(HttpStatus.OK); return Mono.empty(); } return chain.filter(ctx); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/CustomWebFilter.java ================================================ package com.provectus.kafka.ui.config; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilter; import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; @Component public class CustomWebFilter implements WebFilter { @Override public Mono filter(ServerWebExchange exchange, WebFilterChain chain) { final String basePath = exchange.getRequest().getPath().contextPath().value(); final String path = exchange.getRequest().getPath().pathWithinApplication().value(); if (path.startsWith("/ui") || path.equals("") || path.equals("/")) { return chain.filter( exchange.mutate().request( exchange.getRequest().mutate() .path(basePath + "/index.html") .contextPath(basePath) .build() ).build() ); } return chain.filter(exchange); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ReadOnlyModeFilter.java ================================================ package com.provectus.kafka.ui.config; import com.provectus.kafka.ui.exception.ClusterNotFoundException; import com.provectus.kafka.ui.exception.ReadOnlyModeException; import com.provectus.kafka.ui.service.ClustersStorage; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.regex.Pattern; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; import org.springframework.core.annotation.Order; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilter; import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; @Order @Component @RequiredArgsConstructor public class ReadOnlyModeFilter implements WebFilter { private static final Pattern CLUSTER_NAME_REGEX = Pattern.compile("/api/clusters/(?[^/]++)"); private final ClustersStorage clustersStorage; @NotNull @Override public Mono filter(ServerWebExchange exchange, @NotNull WebFilterChain chain) { var isSafeMethod = exchange.getRequest().getMethod() == HttpMethod.GET; if (isSafeMethod) { return chain.filter(exchange); } var path = exchange.getRequest().getPath().pathWithinApplication().value(); var decodedPath = URLDecoder.decode(path, StandardCharsets.UTF_8); var matcher = CLUSTER_NAME_REGEX.matcher(decodedPath); if (!matcher.find()) { return chain.filter(exchange); } var clusterName = matcher.group("clusterName"); var kafkaCluster = clustersStorage.getClusterByName(clusterName) .orElseThrow( () -> new ClusterNotFoundException( String.format("No cluster for name '%s'", clusterName))); if (!kafkaCluster.isReadOnly()) { return chain.filter(exchange); } return Mono.error(ReadOnlyModeException::new); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/WebclientProperties.java ================================================ package com.provectus.kafka.ui.config; import com.provectus.kafka.ui.exception.ValidationException; import javax.annotation.PostConstruct; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.util.unit.DataSize; @Configuration @ConfigurationProperties("webclient") @Data public class WebclientProperties { String maxInMemoryBufferSize; @PostConstruct public void validate() { validateAndSetDefaultBufferSize(); } private void validateAndSetDefaultBufferSize() { if (maxInMemoryBufferSize != null) { try { DataSize.parse(maxInMemoryBufferSize); } catch (Exception e) { throw new ValidationException("Invalid format for webclient.maxInMemoryBufferSize"); } } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AbstractAuthSecurityConfig.java ================================================ package com.provectus.kafka.ui.config.auth; abstract class AbstractAuthSecurityConfig { protected AbstractAuthSecurityConfig() { } protected static final String[] AUTH_WHITELIST = { "/css/**", "/js/**", "/media/**", "/resources/**", "/actuator/health/**", "/actuator/info", "/actuator/prometheus", "/auth", "/login", "/logout", "/oauth2/**", "/static/**" }; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/AuthenticatedUser.java ================================================ package com.provectus.kafka.ui.config.auth; import java.util.Collection; public record AuthenticatedUser(String principal, Collection groups) { } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/BasicAuthSecurityConfig.java ================================================ package com.provectus.kafka.ui.config.auth; import com.provectus.kafka.ui.util.EmptyRedirectStrategy; import java.net.URI; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpMethod; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.web.server.SecurityWebFilterChain; import org.springframework.security.web.server.authentication.RedirectServerAuthenticationSuccessHandler; import org.springframework.security.web.server.authentication.logout.RedirectServerLogoutSuccessHandler; import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatchers; @Configuration @EnableWebFluxSecurity @ConditionalOnProperty(value = "auth.type", havingValue = "LOGIN_FORM") @Slf4j public class BasicAuthSecurityConfig extends AbstractAuthSecurityConfig { public static final String LOGIN_URL = "/auth"; public static final String LOGOUT_URL = "/auth?logout"; @Bean public SecurityWebFilterChain configure(ServerHttpSecurity http) { log.info("Configuring LOGIN_FORM authentication."); final var authHandler = new RedirectServerAuthenticationSuccessHandler(); authHandler.setRedirectStrategy(new EmptyRedirectStrategy()); final var logoutSuccessHandler = new RedirectServerLogoutSuccessHandler(); logoutSuccessHandler.setLogoutSuccessUrl(URI.create(LOGOUT_URL)); return http.authorizeExchange(spec -> spec .pathMatchers(AUTH_WHITELIST) .permitAll() .anyExchange() .authenticated() ) .formLogin(spec -> spec.loginPage(LOGIN_URL).authenticationSuccessHandler(authHandler)) .logout(spec -> spec .logoutSuccessHandler(logoutSuccessHandler) .requiresLogout(ServerWebExchangeMatchers.pathMatchers(HttpMethod.GET, "/logout"))) .csrf(ServerHttpSecurity.CsrfSpec::disable) .build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/DisabledAuthSecurityConfig.java ================================================ package com.provectus.kafka.ui.config.auth; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.web.server.SecurityWebFilterChain; @Configuration @EnableWebFluxSecurity @ConditionalOnProperty(value = "auth.type", havingValue = "DISABLED") @Slf4j public class DisabledAuthSecurityConfig extends AbstractAuthSecurityConfig { @Bean public SecurityWebFilterChain configure(ServerHttpSecurity http, Environment env, ApplicationContext context) { if (env.getProperty("auth.enabled") != null) { log.error("A deprecated property (auth.enabled) is present. " + "Please replace it with 'auth.type' (possible values are: 'LOGIN_FORM', 'DISABLED', 'OAUTH2', 'LDAP') " + "and restart the application."); SpringApplication.exit(context, () -> 1); System.exit(1); } log.warn("Authentication is disabled. Access will be unrestricted."); return http.authorizeExchange(spec -> spec .anyExchange() .permitAll() ) .csrf(ServerHttpSecurity.CsrfSpec::disable) .build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapProperties.java ================================================ package com.provectus.kafka.ui.config.auth; import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("spring.ldap") @Data public class LdapProperties { private String urls; private String base; private String adminUser; private String adminPassword; private String userFilterSearchBase; private String userFilterSearchFilter; private String groupFilterSearchBase; private String groupFilterSearchFilter; private String groupRoleAttribute; @Value("${oauth2.ldap.activeDirectory:false}") private boolean isActiveDirectory; @Value("${oauth2.ldap.aсtiveDirectory.domain:@null}") private String activeDirectoryDomain; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/LdapSecurityConfig.java ================================================ package com.provectus.kafka.ui.config.auth; import static com.provectus.kafka.ui.config.auth.AbstractAuthSecurityConfig.AUTH_WHITELIST; import com.provectus.kafka.ui.service.rbac.AccessControlService; import com.provectus.kafka.ui.service.rbac.extractor.RbacLdapAuthoritiesExtractor; import java.util.Collection; import java.util.List; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.ldap.core.support.LdapContextSource; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.authentication.ProviderManager; import org.springframework.security.authentication.ReactiveAuthenticationManager; import org.springframework.security.authentication.ReactiveAuthenticationManagerAdapter; import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.userdetails.UserDetails; import org.springframework.security.ldap.authentication.AbstractLdapAuthenticationProvider; import org.springframework.security.ldap.authentication.BindAuthenticator; import org.springframework.security.ldap.authentication.LdapAuthenticationProvider; import org.springframework.security.ldap.authentication.ad.ActiveDirectoryLdapAuthenticationProvider; import org.springframework.security.ldap.search.FilterBasedLdapUserSearch; import org.springframework.security.ldap.search.LdapUserSearch; import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; import org.springframework.security.ldap.userdetails.LdapAuthoritiesPopulator; import org.springframework.security.ldap.userdetails.LdapUserDetailsMapper; import org.springframework.security.web.server.SecurityWebFilterChain; @Configuration @EnableWebFluxSecurity @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") @Import(LdapAutoConfiguration.class) @EnableConfigurationProperties(LdapProperties.class) @RequiredArgsConstructor @Slf4j public class LdapSecurityConfig { private final LdapProperties props; @Bean public ReactiveAuthenticationManager authenticationManager(BaseLdapPathContextSource contextSource, LdapAuthoritiesPopulator authoritiesExtractor, AccessControlService acs) { var rbacEnabled = acs.isRbacEnabled(); BindAuthenticator ba = new BindAuthenticator(contextSource); if (props.getBase() != null) { ba.setUserDnPatterns(new String[] {props.getBase()}); } if (props.getUserFilterSearchFilter() != null) { LdapUserSearch userSearch = new FilterBasedLdapUserSearch(props.getUserFilterSearchBase(), props.getUserFilterSearchFilter(), contextSource); ba.setUserSearch(userSearch); } AbstractLdapAuthenticationProvider authenticationProvider; if (!props.isActiveDirectory()) { authenticationProvider = rbacEnabled ? new LdapAuthenticationProvider(ba, authoritiesExtractor) : new LdapAuthenticationProvider(ba); } else { authenticationProvider = new ActiveDirectoryLdapAuthenticationProvider(props.getActiveDirectoryDomain(), props.getUrls()); // TODO Issue #3741 authenticationProvider.setUseAuthenticationRequestCredentials(true); } if (rbacEnabled) { authenticationProvider.setUserDetailsContextMapper(new UserDetailsMapper()); } AuthenticationManager am = new ProviderManager(List.of(authenticationProvider)); return new ReactiveAuthenticationManagerAdapter(am); } @Bean @Primary public BaseLdapPathContextSource contextSource() { LdapContextSource ctx = new LdapContextSource(); ctx.setUrl(props.getUrls()); ctx.setUserDn(props.getAdminUser()); ctx.setPassword(props.getAdminPassword()); ctx.afterPropertiesSet(); return ctx; } @Bean @Primary public DefaultLdapAuthoritiesPopulator ldapAuthoritiesExtractor(ApplicationContext context, BaseLdapPathContextSource contextSource, AccessControlService acs) { var rbacEnabled = acs != null && acs.isRbacEnabled(); DefaultLdapAuthoritiesPopulator extractor; if (rbacEnabled) { extractor = new RbacLdapAuthoritiesExtractor(context, contextSource, props.getGroupFilterSearchBase()); } else { extractor = new DefaultLdapAuthoritiesPopulator(contextSource, props.getGroupFilterSearchBase()); } Optional.ofNullable(props.getGroupFilterSearchFilter()).ifPresent(extractor::setGroupSearchFilter); extractor.setRolePrefix(""); extractor.setConvertToUpperCase(false); extractor.setSearchSubtree(true); return extractor; } @Bean public SecurityWebFilterChain configureLdap(ServerHttpSecurity http) { log.info("Configuring LDAP authentication."); if (props.isActiveDirectory()) { log.info("Active Directory support for LDAP has been enabled."); } return http.authorizeExchange(spec -> spec .pathMatchers(AUTH_WHITELIST) .permitAll() .anyExchange() .authenticated() ) .formLogin(Customizer.withDefaults()) .logout(Customizer.withDefaults()) .csrf(ServerHttpSecurity.CsrfSpec::disable) .build(); } private static class UserDetailsMapper extends LdapUserDetailsMapper { @Override public UserDetails mapUserFromContext(DirContextOperations ctx, String username, Collection authorities) { UserDetails userDetails = super.mapUserFromContext(ctx, username, authorities); return new RbacLdapUser(userDetails); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthProperties.java ================================================ package com.provectus.kafka.ui.config.auth; import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.util.Assert; @ConfigurationProperties("auth.oauth2") @Data public class OAuthProperties { private Map client = new HashMap<>(); @PostConstruct public void init() { getClient().values().forEach((provider) -> { if (provider.getCustomParams() == null) { provider.setCustomParams(Collections.emptyMap()); } if (provider.getScope() == null) { provider.setScope(Collections.emptySet()); } }); getClient().values().forEach(this::validateProvider); } private void validateProvider(final OAuth2Provider provider) { Assert.hasText(provider.getClientId(), "Client id must not be empty."); Assert.hasText(provider.getProvider(), "Provider name must not be empty"); } @Data public static class OAuth2Provider { private String provider; private String clientId; private String clientSecret; private String clientName; private String redirectUri; private String authorizationGrantType; private Set scope; private String issuerUri; private String authorizationUri; private String tokenUri; private String userInfoUri; private String jwkSetUri; private String userNameAttribute; private Map customParams; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthPropertiesConverter.java ================================================ package com.provectus.kafka.ui.config.auth; import static com.provectus.kafka.ui.config.auth.OAuthProperties.OAuth2Provider; import static org.springframework.boot.autoconfigure.security.oauth2.client.OAuth2ClientProperties.Provider; import static org.springframework.boot.autoconfigure.security.oauth2.client.OAuth2ClientProperties.Registration; import java.util.Optional; import java.util.Set; import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.autoconfigure.security.oauth2.client.OAuth2ClientProperties; import org.springframework.security.config.oauth2.client.CommonOAuth2Provider; @NoArgsConstructor(access = AccessLevel.PRIVATE) public final class OAuthPropertiesConverter { private static final String TYPE = "type"; private static final String GOOGLE = "google"; public static final String DUMMY = "dummy"; public static OAuth2ClientProperties convertProperties(final OAuthProperties properties) { final var result = new OAuth2ClientProperties(); properties.getClient().forEach((key, provider) -> { var registration = new Registration(); registration.setClientId(provider.getClientId()); registration.setClientSecret(provider.getClientSecret()); registration.setClientName(provider.getClientName()); registration.setScope(Optional.ofNullable(provider.getScope()).orElse(Set.of())); registration.setRedirectUri(provider.getRedirectUri()); registration.setAuthorizationGrantType(provider.getAuthorizationGrantType()); result.getRegistration().put(key, registration); var clientProvider = new Provider(); applyCustomTransformations(provider); clientProvider.setAuthorizationUri(provider.getAuthorizationUri()); clientProvider.setIssuerUri(provider.getIssuerUri()); clientProvider.setJwkSetUri(provider.getJwkSetUri()); clientProvider.setTokenUri(provider.getTokenUri()); clientProvider.setUserInfoUri(provider.getUserInfoUri()); clientProvider.setUserNameAttribute(provider.getUserNameAttribute()); result.getProvider().put(key, clientProvider); }); return result; } private static void applyCustomTransformations(OAuth2Provider provider) { applyGoogleTransformations(provider); } private static void applyGoogleTransformations(OAuth2Provider provider) { if (!isGoogle(provider)) { return; } String allowedDomain = provider.getCustomParams().get("allowedDomain"); if (StringUtils.isEmpty(allowedDomain)) { return; } String authorizationUri = CommonOAuth2Provider.GOOGLE .getBuilder(DUMMY) .clientId(DUMMY) .build() .getProviderDetails() .getAuthorizationUri(); final String newUri = authorizationUri + "?hd=" + allowedDomain; provider.setAuthorizationUri(newUri); } private static boolean isGoogle(OAuth2Provider provider) { return GOOGLE.equalsIgnoreCase(provider.getCustomParams().get(TYPE)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthSecurityConfig.java ================================================ package com.provectus.kafka.ui.config.auth; import com.provectus.kafka.ui.config.auth.logout.OAuthLogoutSuccessHandler; import com.provectus.kafka.ui.service.rbac.AccessControlService; import com.provectus.kafka.ui.service.rbac.extractor.ProviderAuthorityExtractor; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.jetbrains.annotations.Nullable; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.security.oauth2.client.OAuth2ClientProperties; import org.springframework.boot.autoconfigure.security.oauth2.client.OAuth2ClientPropertiesMapper; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.method.configuration.EnableReactiveMethodSecurity; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.oauth2.client.oidc.userinfo.OidcReactiveOAuth2UserService; import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserRequest; import org.springframework.security.oauth2.client.oidc.web.server.logout.OidcClientInitiatedServerLogoutSuccessHandler; import org.springframework.security.oauth2.client.registration.ClientRegistration; import org.springframework.security.oauth2.client.registration.InMemoryReactiveClientRegistrationRepository; import org.springframework.security.oauth2.client.registration.ReactiveClientRegistrationRepository; import org.springframework.security.oauth2.client.userinfo.DefaultReactiveOAuth2UserService; import org.springframework.security.oauth2.client.userinfo.OAuth2UserRequest; import org.springframework.security.oauth2.client.userinfo.ReactiveOAuth2UserService; import org.springframework.security.oauth2.core.oidc.user.OidcUser; import org.springframework.security.oauth2.core.user.OAuth2User; import org.springframework.security.web.server.SecurityWebFilterChain; import org.springframework.security.web.server.authentication.logout.ServerLogoutSuccessHandler; import reactor.core.publisher.Mono; @Configuration @ConditionalOnProperty(value = "auth.type", havingValue = "OAUTH2") @EnableConfigurationProperties(OAuthProperties.class) @EnableWebFluxSecurity @EnableReactiveMethodSecurity @RequiredArgsConstructor @Log4j2 public class OAuthSecurityConfig extends AbstractAuthSecurityConfig { private final OAuthProperties properties; @Bean public SecurityWebFilterChain configure(ServerHttpSecurity http, OAuthLogoutSuccessHandler logoutHandler) { log.info("Configuring OAUTH2 authentication."); return http.authorizeExchange(spec -> spec .pathMatchers(AUTH_WHITELIST) .permitAll() .anyExchange() .authenticated() ) .oauth2Login(Customizer.withDefaults()) .logout(spec -> spec.logoutSuccessHandler(logoutHandler)) .csrf(ServerHttpSecurity.CsrfSpec::disable) .build(); } @Bean public ReactiveOAuth2UserService customOidcUserService(AccessControlService acs) { final OidcReactiveOAuth2UserService delegate = new OidcReactiveOAuth2UserService(); return request -> delegate.loadUser(request) .flatMap(user -> { var provider = getProviderByProviderId(request.getClientRegistration().getRegistrationId()); final var extractor = getExtractor(provider, acs); if (extractor == null) { return Mono.just(user); } return extractor.extract(acs, user, Map.of("request", request, "provider", provider)) .map(groups -> new RbacOidcUser(user, groups)); }); } @Bean public ReactiveOAuth2UserService customOauth2UserService(AccessControlService acs) { final DefaultReactiveOAuth2UserService delegate = new DefaultReactiveOAuth2UserService(); return request -> delegate.loadUser(request) .flatMap(user -> { var provider = getProviderByProviderId(request.getClientRegistration().getRegistrationId()); final var extractor = getExtractor(provider, acs); if (extractor == null) { return Mono.just(user); } return extractor.extract(acs, user, Map.of("request", request, "provider", provider)) .map(groups -> new RbacOAuth2User(user, groups)); }); } @Bean public InMemoryReactiveClientRegistrationRepository clientRegistrationRepository() { final OAuth2ClientProperties props = OAuthPropertiesConverter.convertProperties(properties); final List registrations = new ArrayList<>(new OAuth2ClientPropertiesMapper(props).asClientRegistrations().values()); if (registrations.isEmpty()) { throw new IllegalArgumentException("OAuth2 authentication is enabled but no providers specified."); } return new InMemoryReactiveClientRegistrationRepository(registrations); } @Bean public ServerLogoutSuccessHandler defaultOidcLogoutHandler(final ReactiveClientRegistrationRepository repository) { return new OidcClientInitiatedServerLogoutSuccessHandler(repository); } @Nullable private ProviderAuthorityExtractor getExtractor(final OAuthProperties.OAuth2Provider provider, AccessControlService acs) { Optional extractor = acs.getOauthExtractors() .stream() .filter(e -> e.isApplicable(provider.getProvider(), provider.getCustomParams())) .findFirst(); return extractor.orElse(null); } private OAuthProperties.OAuth2Provider getProviderByProviderId(final String providerId) { return properties.getClient().get(providerId); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacLdapUser.java ================================================ package com.provectus.kafka.ui.config.auth; import java.util.Collection; import java.util.stream.Collectors; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.userdetails.UserDetails; public class RbacLdapUser implements UserDetails, RbacUser { private final UserDetails userDetails; public RbacLdapUser(UserDetails userDetails) { this.userDetails = userDetails; } @Override public String name() { return userDetails.getUsername(); } @Override public Collection groups() { return userDetails.getAuthorities().stream().map(GrantedAuthority::getAuthority).collect(Collectors.toSet()); } @Override public Collection getAuthorities() { return userDetails.getAuthorities(); } @Override public String getPassword() { return userDetails.getPassword(); } @Override public String getUsername() { return userDetails.getUsername(); } @Override public boolean isAccountNonExpired() { return userDetails.isAccountNonExpired(); } @Override public boolean isAccountNonLocked() { return userDetails.isAccountNonLocked(); } @Override public boolean isCredentialsNonExpired() { return userDetails.isCredentialsNonExpired(); } @Override public boolean isEnabled() { return userDetails.isEnabled(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacOAuth2User.java ================================================ package com.provectus.kafka.ui.config.auth; import java.util.Collection; import java.util.Map; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.oauth2.core.user.OAuth2User; public record RbacOAuth2User(OAuth2User user, Collection groups) implements RbacUser, OAuth2User { @Override public Map getAttributes() { return user.getAttributes(); } @Override public Collection getAuthorities() { return user.getAuthorities(); } @Override public String getName() { return user.getName(); } @Override public String name() { return user.getName(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacOidcUser.java ================================================ package com.provectus.kafka.ui.config.auth; import java.util.Collection; import java.util.Map; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.oauth2.core.oidc.OidcIdToken; import org.springframework.security.oauth2.core.oidc.OidcUserInfo; import org.springframework.security.oauth2.core.oidc.user.OidcUser; public record RbacOidcUser(OidcUser user, Collection groups) implements RbacUser, OidcUser { @Override public Map getClaims() { return user.getClaims(); } @Override public OidcUserInfo getUserInfo() { return user.getUserInfo(); } @Override public OidcIdToken getIdToken() { return user.getIdToken(); } @Override public Map getAttributes() { return user.getAttributes(); } @Override public Collection getAuthorities() { return user.getAuthorities(); } @Override public String getName() { return user.getName(); } @Override public String name() { return user.getName(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RbacUser.java ================================================ package com.provectus.kafka.ui.config.auth; import java.util.Collection; public interface RbacUser { String name(); Collection groups(); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/RoleBasedAccessControlProperties.java ================================================ package com.provectus.kafka.ui.config.auth; import com.provectus.kafka.ui.model.rbac.Role; import java.util.ArrayList; import java.util.List; import javax.annotation.PostConstruct; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties("rbac") public class RoleBasedAccessControlProperties { private final List roles = new ArrayList<>(); @PostConstruct public void init() { roles.forEach(Role::validate); } public List getRoles() { return roles; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/ActiveDirectoryCondition.java ================================================ package com.provectus.kafka.ui.config.auth.condition; import org.springframework.boot.autoconfigure.condition.AllNestedConditions; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; public class ActiveDirectoryCondition extends AllNestedConditions { public ActiveDirectoryCondition() { super(ConfigurationPhase.PARSE_CONFIGURATION); } @ConditionalOnProperty(value = "auth.type", havingValue = "LDAP") public static class OnAuthType { } @ConditionalOnProperty(value = "${oauth2.ldap.activeDirectory}:false", havingValue = "true", matchIfMissing = false) public static class OnActiveDirectory { } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/condition/CognitoCondition.java ================================================ package com.provectus.kafka.ui.config.auth.condition; import com.provectus.kafka.ui.service.rbac.AbstractProviderCondition; import org.jetbrains.annotations.NotNull; import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; import org.springframework.core.type.AnnotatedTypeMetadata; public class CognitoCondition extends AbstractProviderCondition implements Condition { @Override public boolean matches(final ConditionContext context, final @NotNull AnnotatedTypeMetadata metadata) { return getRegisteredProvidersTypes(context.getEnvironment()).stream().anyMatch(a -> a.equalsIgnoreCase("cognito")); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/logout/CognitoLogoutSuccessHandler.java ================================================ package com.provectus.kafka.ui.config.auth.logout; import com.provectus.kafka.ui.config.auth.OAuthProperties; import com.provectus.kafka.ui.config.auth.condition.CognitoCondition; import com.provectus.kafka.ui.model.rbac.provider.Provider; import java.net.URI; import java.nio.charset.StandardCharsets; import org.springframework.context.annotation.Conditional; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.security.core.Authentication; import org.springframework.security.web.server.WebFilterExchange; import org.springframework.security.web.util.UrlUtils; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import org.springframework.web.server.WebSession; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Mono; @Component @Conditional(CognitoCondition.class) public class CognitoLogoutSuccessHandler implements LogoutSuccessHandler { @Override public boolean isApplicable(String provider) { return Provider.Name.COGNITO.equalsIgnoreCase(provider); } @Override public Mono handle(WebFilterExchange exchange, Authentication authentication, OAuthProperties.OAuth2Provider provider) { final ServerHttpResponse response = exchange.getExchange().getResponse(); response.setStatusCode(HttpStatus.FOUND); final var requestUri = exchange.getExchange().getRequest().getURI(); final var fullUrl = UrlUtils.buildFullRequestUrl(requestUri.getScheme(), requestUri.getHost(), requestUri.getPort(), requestUri.getPath(), requestUri.getQuery()); final UriComponents baseUrl = UriComponentsBuilder .fromHttpUrl(fullUrl) .replacePath("/") .replaceQuery(null) .fragment(null) .build(); Assert.isTrue(provider.getCustomParams().containsKey("logoutUrl"), "Custom params should contain 'logoutUrl'"); final var uri = UriComponentsBuilder .fromUri(URI.create(provider.getCustomParams().get("logoutUrl"))) .queryParam("client_id", provider.getClientId()) .queryParam("logout_uri", baseUrl) .encode(StandardCharsets.UTF_8) .build() .toUri(); response.getHeaders().setLocation(uri); return exchange.getExchange().getSession().flatMap(WebSession::invalidate); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/logout/LogoutSuccessHandler.java ================================================ package com.provectus.kafka.ui.config.auth.logout; import com.provectus.kafka.ui.config.auth.OAuthProperties; import org.springframework.security.core.Authentication; import org.springframework.security.web.server.WebFilterExchange; import reactor.core.publisher.Mono; public interface LogoutSuccessHandler { boolean isApplicable(final String provider); Mono handle(final WebFilterExchange exchange, final Authentication authentication, final OAuthProperties.OAuth2Provider provider); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/logout/OAuthLogoutSuccessHandler.java ================================================ package com.provectus.kafka.ui.config.auth.logout; import com.provectus.kafka.ui.config.auth.OAuthProperties; import java.util.List; import java.util.Optional; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.core.Authentication; import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; import org.springframework.security.web.server.WebFilterExchange; import org.springframework.security.web.server.authentication.logout.ServerLogoutSuccessHandler; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; @Component @ConditionalOnProperty(value = "auth.type", havingValue = "OAUTH2") public class OAuthLogoutSuccessHandler implements ServerLogoutSuccessHandler { private final OAuthProperties properties; private final List logoutSuccessHandlers; private final ServerLogoutSuccessHandler defaultOidcLogoutHandler; public OAuthLogoutSuccessHandler(final OAuthProperties properties, final List logoutSuccessHandlers, final @Qualifier("defaultOidcLogoutHandler") ServerLogoutSuccessHandler handler) { this.properties = properties; this.logoutSuccessHandlers = logoutSuccessHandlers; this.defaultOidcLogoutHandler = handler; } @Override public Mono onLogoutSuccess(final WebFilterExchange exchange, final Authentication authentication) { final OAuth2AuthenticationToken oauthToken = (OAuth2AuthenticationToken) authentication; final String providerId = oauthToken.getAuthorizedClientRegistrationId(); final OAuthProperties.OAuth2Provider oAuth2Provider = properties.getClient().get(providerId); return getLogoutHandler(oAuth2Provider.getProvider()) .map(handler -> handler.handle(exchange, authentication, oAuth2Provider)) .orElseGet(() -> defaultOidcLogoutHandler.onLogoutSuccess(exchange, authentication)); } private Optional getLogoutHandler(final String provider) { return logoutSuccessHandlers.stream() .filter(h -> h.isApplicable(provider)) .findFirst(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AbstractController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.exception.ClusterNotFoundException; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.ClustersStorage; import com.provectus.kafka.ui.service.audit.AuditService; import com.provectus.kafka.ui.service.rbac.AccessControlService; import org.springframework.beans.factory.annotation.Autowired; import reactor.core.publisher.Mono; import reactor.core.publisher.Signal; public abstract class AbstractController { protected ClustersStorage clustersStorage; protected AccessControlService accessControlService; protected AuditService auditService; protected KafkaCluster getCluster(String name) { return clustersStorage.getClusterByName(name) .orElseThrow(() -> new ClusterNotFoundException( String.format("Cluster with name '%s' not found", name))); } protected Mono validateAccess(AccessContext context) { return accessControlService.validateAccess(context); } protected void audit(AccessContext acxt, Signal sig) { auditService.audit(acxt, sig); } @Autowired public void setClustersStorage(ClustersStorage clustersStorage) { this.clustersStorage = clustersStorage; } @Autowired public void setAccessControlService(AccessControlService accessControlService) { this.accessControlService = accessControlService; } @Autowired public void setAuditService(AuditService auditService) { this.auditService = auditService; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AccessController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.AuthorizationApi; import com.provectus.kafka.ui.model.ActionDTO; import com.provectus.kafka.ui.model.AuthenticationInfoDTO; import com.provectus.kafka.ui.model.ResourceTypeDTO; import com.provectus.kafka.ui.model.UserInfoDTO; import com.provectus.kafka.ui.model.UserPermissionDTO; import com.provectus.kafka.ui.model.rbac.Permission; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.security.Principal; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.ReactiveSecurityContextHolder; import org.springframework.security.core.context.SecurityContext; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class AccessController implements AuthorizationApi { private final AccessControlService accessControlService; public Mono> getUserAuthInfo(ServerWebExchange exchange) { Mono> permissions = accessControlService.getUser() .map(user -> accessControlService.getRoles() .stream() .filter(role -> user.groups().contains(role.getName())) .map(role -> mapPermissions(role.getPermissions(), role.getClusters())) .flatMap(Collection::stream) .toList() ) .switchIfEmpty(Mono.just(Collections.emptyList())); Mono userName = ReactiveSecurityContextHolder.getContext() .map(SecurityContext::getAuthentication) .map(Principal::getName); return userName .zipWith(permissions) .map(data -> { var dto = new AuthenticationInfoDTO(accessControlService.isRbacEnabled()); dto.setUserInfo(new UserInfoDTO(data.getT1(), data.getT2())); return dto; }) .switchIfEmpty(Mono.just(new AuthenticationInfoDTO(accessControlService.isRbacEnabled()))) .map(ResponseEntity::ok); } private List mapPermissions(List permissions, List clusters) { return permissions .stream() .map(permission -> { UserPermissionDTO dto = new UserPermissionDTO(); dto.setClusters(clusters); dto.setResource(ResourceTypeDTO.fromValue(permission.getResource().toString().toUpperCase())); dto.setValue(permission.getValue()); dto.setActions(permission.getActions() .stream() .map(String::toUpperCase) .map(this::mapAction) .filter(Objects::nonNull) .toList()); return dto; }) .toList(); } @Nullable private ActionDTO mapAction(String name) { try { return ActionDTO.fromValue(name); } catch (IllegalArgumentException e) { log.warn("Unknown Action [{}], skipping", name); return null; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AclsController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.AclsApi; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.model.CreateConsumerAclDTO; import com.provectus.kafka.ui.model.CreateProducerAclDTO; import com.provectus.kafka.ui.model.CreateStreamAppAclDTO; import com.provectus.kafka.ui.model.KafkaAclDTO; import com.provectus.kafka.ui.model.KafkaAclNamePatternTypeDTO; import com.provectus.kafka.ui.model.KafkaAclResourceTypeDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.service.acl.AclsService; import java.util.Optional; import lombok.RequiredArgsConstructor; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor public class AclsController extends AbstractController implements AclsApi { private final AclsService aclsService; @Override public Mono> createAcl(String clusterName, Mono kafkaAclDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("createAcl") .build(); return validateAccess(context) .then(kafkaAclDto) .map(ClusterMapper::toAclBinding) .flatMap(binding -> aclsService.createAcl(getCluster(clusterName), binding)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> deleteAcl(String clusterName, Mono kafkaAclDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("deleteAcl") .build(); return validateAccess(context) .then(kafkaAclDto) .map(ClusterMapper::toAclBinding) .flatMap(binding -> aclsService.deleteAcl(getCluster(clusterName), binding)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono>> listAcls(String clusterName, KafkaAclResourceTypeDTO resourceTypeDto, String resourceName, KafkaAclNamePatternTypeDTO namePatternTypeDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.VIEW) .operationName("listAcls") .build(); var resourceType = Optional.ofNullable(resourceTypeDto) .map(ClusterMapper::mapAclResourceTypeDto) .orElse(ResourceType.ANY); var namePatternType = Optional.ofNullable(namePatternTypeDto) .map(ClusterMapper::mapPatternTypeDto) .orElse(PatternType.ANY); var filter = new ResourcePatternFilter(resourceType, resourceName, namePatternType); return validateAccess(context).then( Mono.just( ResponseEntity.ok( aclsService.listAcls(getCluster(clusterName), filter) .map(ClusterMapper::toKafkaAclDto))) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getAclAsCsv(String clusterName, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.VIEW) .operationName("getAclAsCsv") .build(); return validateAccess(context).then( aclsService.getAclAsCsvString(getCluster(clusterName)) .map(ResponseEntity::ok) .flatMap(Mono::just) .doOnEach(sig -> audit(context, sig)) ); } @Override public Mono> syncAclsCsv(String clusterName, Mono csvMono, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("syncAclsCsv") .build(); return validateAccess(context) .then(csvMono) .flatMap(csv -> aclsService.syncAclWithAclCsv(getCluster(clusterName), csv)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> createConsumerAcl(String clusterName, Mono createConsumerAclDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("createConsumerAcl") .build(); return validateAccess(context) .then(createConsumerAclDto) .flatMap(req -> aclsService.createConsumerAcl(getCluster(clusterName), req)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> createProducerAcl(String clusterName, Mono createProducerAclDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("createProducerAcl") .build(); return validateAccess(context) .then(createProducerAclDto) .flatMap(req -> aclsService.createProducerAcl(getCluster(clusterName), req)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> createStreamAppAcl(String clusterName, Mono createStreamAppAclDto, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .aclActions(AclAction.EDIT) .operationName("createStreamAppAcl") .build(); return validateAccess(context) .then(createStreamAppAclDto) .flatMap(req -> aclsService.createStreamAppAcl(getCluster(clusterName), req)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ApplicationConfigController.java ================================================ package com.provectus.kafka.ui.controller; import static com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction.EDIT; import static com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction.VIEW; import com.provectus.kafka.ui.api.ApplicationConfigApi; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.ApplicationConfigDTO; import com.provectus.kafka.ui.model.ApplicationConfigPropertiesDTO; import com.provectus.kafka.ui.model.ApplicationConfigValidationDTO; import com.provectus.kafka.ui.model.ApplicationInfoDTO; import com.provectus.kafka.ui.model.ClusterConfigValidationDTO; import com.provectus.kafka.ui.model.RestartRequestDTO; import com.provectus.kafka.ui.model.UploadedFileInfoDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.ApplicationInfoService; import com.provectus.kafka.ui.service.KafkaClusterFactory; import com.provectus.kafka.ui.util.ApplicationRestarter; import com.provectus.kafka.ui.util.DynamicConfigOperations; import com.provectus.kafka.ui.util.DynamicConfigOperations.PropertiesStructure; import java.util.Map; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.mapstruct.Mapper; import org.mapstruct.factory.Mappers; import org.springframework.http.ResponseEntity; import org.springframework.http.codec.multipart.FilePart; import org.springframework.http.codec.multipart.Part; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Slf4j @RestController @RequiredArgsConstructor public class ApplicationConfigController extends AbstractController implements ApplicationConfigApi { private static final PropertiesMapper MAPPER = Mappers.getMapper(PropertiesMapper.class); @Mapper interface PropertiesMapper { PropertiesStructure fromDto(ApplicationConfigPropertiesDTO dto); ApplicationConfigPropertiesDTO toDto(PropertiesStructure propertiesStructure); } private final DynamicConfigOperations dynamicConfigOperations; private final ApplicationRestarter restarter; private final KafkaClusterFactory kafkaClusterFactory; private final ApplicationInfoService applicationInfoService; @Override public Mono> getApplicationInfo(ServerWebExchange exchange) { return Mono.just(applicationInfoService.getApplicationInfo()).map(ResponseEntity::ok); } @Override public Mono> getCurrentConfig(ServerWebExchange exchange) { var context = AccessContext.builder() .applicationConfigActions(VIEW) .operationName("getCurrentConfig") .build(); return validateAccess(context) .then(Mono.fromSupplier(() -> ResponseEntity.ok( new ApplicationConfigDTO() .properties(MAPPER.toDto(dynamicConfigOperations.getCurrentProperties())) ))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> restartWithConfig(Mono restartRequestDto, ServerWebExchange exchange) { var context = AccessContext.builder() .applicationConfigActions(EDIT) .operationName("restartWithConfig") .build(); return validateAccess(context) .then(restartRequestDto) .doOnNext(restartDto -> { var newConfig = MAPPER.fromDto(restartDto.getConfig().getProperties()); dynamicConfigOperations.persist(newConfig); }) .doOnEach(sig -> audit(context, sig)) .doOnSuccess(dto -> restarter.requestRestart()) .map(dto -> ResponseEntity.ok().build()); } @Override public Mono> uploadConfigRelatedFile(Flux fileFlux, ServerWebExchange exchange) { var context = AccessContext.builder() .applicationConfigActions(EDIT) .operationName("uploadConfigRelatedFile") .build(); return validateAccess(context) .then(fileFlux.single()) .flatMap(file -> dynamicConfigOperations.uploadConfigRelatedFile((FilePart) file) .map(path -> new UploadedFileInfoDTO().location(path.toString())) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> validateConfig(Mono configDto, ServerWebExchange exchange) { var context = AccessContext.builder() .applicationConfigActions(EDIT) .operationName("validateConfig") .build(); return validateAccess(context) .then(configDto) .flatMap(config -> { PropertiesStructure newConfig = MAPPER.fromDto(config.getProperties()); ClustersProperties clustersProperties = newConfig.getKafka(); return validateClustersConfig(clustersProperties) .map(validations -> new ApplicationConfigValidationDTO().clusters(validations)); }) .map(ResponseEntity::ok) .doOnEach(sig -> audit(context, sig)); } private Mono> validateClustersConfig( @Nullable ClustersProperties properties) { if (properties == null || properties.getClusters() == null) { return Mono.just(Map.of()); } properties.validateAndSetDefaults(); return Flux.fromIterable(properties.getClusters()) .flatMap(c -> kafkaClusterFactory.validate(c).map(v -> Tuples.of(c.getName(), v))) .collectMap(Tuple2::getT1, Tuple2::getT2); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/AuthController.java ================================================ package com.provectus.kafka.ui.controller; import java.nio.charset.Charset; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.security.web.server.csrf.CsrfToken; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class AuthController { @GetMapping(value = "/auth", produces = {"text/html"}) public Mono getAuth(ServerWebExchange exchange) { Mono token = exchange.getAttributeOrDefault(CsrfToken.class.getName(), Mono.empty()); return token .map(AuthController::csrfToken) .defaultIfEmpty("") .map(csrfTokenHtmlInput -> createPage(exchange, csrfTokenHtmlInput)); } private byte[] createPage(ServerWebExchange exchange, String csrfTokenHtmlInput) { MultiValueMap queryParams = exchange.getRequest() .getQueryParams(); String contextPath = exchange.getRequest().getPath().contextPath().value(); String page = "\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " Please sign in\n" + " \n" + " \n" + " \n" + " \n" + "
\n" + formLogin(queryParams, contextPath, csrfTokenHtmlInput) + "
\n" + " \n" + ""; return page.getBytes(Charset.defaultCharset()); } private String formLogin( MultiValueMap queryParams, String contextPath, String csrfTokenHtmlInput) { boolean isError = queryParams.containsKey("error"); boolean isLogoutSuccess = queryParams.containsKey("logout"); return "
\n" + " \n" + createError(isError) + createLogoutSuccess(isLogoutSuccess) + "

\n" + " \n" + " \n" + "

\n" + "

\n" + " \n" + " \n" + "

\n" + csrfTokenHtmlInput + " \n" + "
\n"; } private static String csrfToken(CsrfToken token) { return " \n"; } private static String createError(boolean isError) { return isError ? "
Invalid credentials
" : ""; } private static String createLogoutSuccess(boolean isLogoutSuccess) { return isLogoutSuccess ? "
You have been signed out
" : ""; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/BrokersController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.BrokersApi; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.model.BrokerConfigDTO; import com.provectus.kafka.ui.model.BrokerConfigItemDTO; import com.provectus.kafka.ui.model.BrokerDTO; import com.provectus.kafka.ui.model.BrokerLogdirUpdateDTO; import com.provectus.kafka.ui.model.BrokerMetricsDTO; import com.provectus.kafka.ui.model.BrokersLogdirsDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.service.BrokerService; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class BrokersController extends AbstractController implements BrokersApi { private static final String BROKER_ID = "brokerId"; private final BrokerService brokerService; private final ClusterMapper clusterMapper; @Override public Mono>> getBrokers(String clusterName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .operationName("getBrokers") .build(); var job = brokerService.getBrokers(getCluster(clusterName)).map(clusterMapper::toBrokerDto); return validateAccess(context) .thenReturn(ResponseEntity.ok(job)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> getBrokersMetrics(String clusterName, Integer id, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .operationName("getBrokersMetrics") .operationParams(Map.of("id", id)) .build(); return validateAccess(context) .then( brokerService.getBrokerMetrics(getCluster(clusterName), id) .map(clusterMapper::toBrokerMetrics) .map(ResponseEntity::ok) .onErrorReturn(ResponseEntity.notFound().build()) ) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getAllBrokersLogdirs(String clusterName, @Nullable List brokers, ServerWebExchange exchange) { List brokerIds = brokers == null ? List.of() : brokers; var context = AccessContext.builder() .cluster(clusterName) .operationName("getAllBrokersLogdirs") .operationParams(Map.of("brokerIds", brokerIds)) .build(); return validateAccess(context) .thenReturn(ResponseEntity.ok( brokerService.getAllBrokersLogdirs(getCluster(clusterName), brokerIds))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getBrokerConfig(String clusterName, Integer id, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .clusterConfigActions(ClusterConfigAction.VIEW) .operationName("getBrokerConfig") .operationParams(Map.of(BROKER_ID, id)) .build(); return validateAccess(context).thenReturn( ResponseEntity.ok( brokerService.getBrokerConfig(getCluster(clusterName), id) .map(clusterMapper::toBrokerConfig)) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateBrokerTopicPartitionLogDir(String clusterName, Integer id, Mono brokerLogdir, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .clusterConfigActions(ClusterConfigAction.VIEW, ClusterConfigAction.EDIT) .operationName("updateBrokerTopicPartitionLogDir") .operationParams(Map.of(BROKER_ID, id)) .build(); return validateAccess(context).then( brokerLogdir .flatMap(bld -> brokerService.updateBrokerLogDir(getCluster(clusterName), id, bld)) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateBrokerConfigByName(String clusterName, Integer id, String name, Mono brokerConfig, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .clusterConfigActions(ClusterConfigAction.VIEW, ClusterConfigAction.EDIT) .operationName("updateBrokerConfigByName") .operationParams(Map.of(BROKER_ID, id)) .build(); return validateAccess(context).then( brokerConfig .flatMap(bci -> brokerService.updateBrokerConfigByName( getCluster(clusterName), id, name, bci.getValue())) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ClustersController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.ClustersApi; import com.provectus.kafka.ui.model.ClusterDTO; import com.provectus.kafka.ui.model.ClusterMetricsDTO; import com.provectus.kafka.ui.model.ClusterStatsDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.ClusterService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class ClustersController extends AbstractController implements ClustersApi { private final ClusterService clusterService; @Override public Mono>> getClusters(ServerWebExchange exchange) { Flux job = Flux.fromIterable(clusterService.getClusters()) .filterWhen(accessControlService::isClusterAccessible); return Mono.just(ResponseEntity.ok(job)); } @Override public Mono> getClusterMetrics(String clusterName, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .operationName("getClusterMetrics") .build(); return validateAccess(context) .then( clusterService.getClusterMetrics(getCluster(clusterName)) .map(ResponseEntity::ok) .onErrorReturn(ResponseEntity.notFound().build()) ) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> getClusterStats(String clusterName, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .operationName("getClusterStats") .build(); return validateAccess(context) .then( clusterService.getClusterStats(getCluster(clusterName)) .map(ResponseEntity::ok) .onErrorReturn(ResponseEntity.notFound().build()) ) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateClusterInfo(String clusterName, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .operationName("updateClusterInfo") .build(); return validateAccess(context) .then(clusterService.updateCluster(getCluster(clusterName)).map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/ConsumerGroupsController.java ================================================ package com.provectus.kafka.ui.controller; import static com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction.DELETE; import static com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction.RESET_OFFSETS; import static com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction.VIEW; import static java.util.stream.Collectors.toMap; import com.provectus.kafka.ui.api.ConsumerGroupsApi; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.mapper.ConsumerGroupMapper; import com.provectus.kafka.ui.model.ConsumerGroupDTO; import com.provectus.kafka.ui.model.ConsumerGroupDetailsDTO; import com.provectus.kafka.ui.model.ConsumerGroupOffsetsResetDTO; import com.provectus.kafka.ui.model.ConsumerGroupOrderingDTO; import com.provectus.kafka.ui.model.ConsumerGroupsPageResponseDTO; import com.provectus.kafka.ui.model.PartitionOffsetDTO; import com.provectus.kafka.ui.model.SortOrderDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import com.provectus.kafka.ui.service.ConsumerGroupService; import com.provectus.kafka.ui.service.OffsetsResetService; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class ConsumerGroupsController extends AbstractController implements ConsumerGroupsApi { private final ConsumerGroupService consumerGroupService; private final OffsetsResetService offsetsResetService; @Value("${consumer.groups.page.size:25}") private int defaultConsumerGroupsPageSize; @Override public Mono> deleteConsumerGroup(String clusterName, String id, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .consumerGroup(id) .consumerGroupActions(DELETE) .operationName("deleteConsumerGroup") .build(); return validateAccess(context) .then(consumerGroupService.deleteConsumerGroupById(getCluster(clusterName), id)) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> getConsumerGroup(String clusterName, String consumerGroupId, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .consumerGroup(consumerGroupId) .consumerGroupActions(VIEW) .operationName("getConsumerGroup") .build(); return validateAccess(context) .then(consumerGroupService.getConsumerGroupDetail(getCluster(clusterName), consumerGroupId) .map(ConsumerGroupMapper::toDetailsDto) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getTopicConsumerGroups(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(TopicAction.VIEW) .operationName("getTopicConsumerGroups") .build(); Mono>> job = consumerGroupService.getConsumerGroupsForTopic(getCluster(clusterName), topicName) .flatMapMany(Flux::fromIterable) .filterWhen(cg -> accessControlService.isConsumerGroupAccessible(cg.getGroupId(), clusterName)) .map(ConsumerGroupMapper::toDto) .collectList() .map(Flux::fromIterable) .map(ResponseEntity::ok) .switchIfEmpty(Mono.just(ResponseEntity.notFound().build())); return validateAccess(context) .then(job) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> getConsumerGroupsPage( String clusterName, Integer page, Integer perPage, String search, ConsumerGroupOrderingDTO orderBy, SortOrderDTO sortOrderDto, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) // consumer group access validation is within the service .operationName("getConsumerGroupsPage") .build(); return validateAccess(context).then( consumerGroupService.getConsumerGroupsPage( getCluster(clusterName), Optional.ofNullable(page).filter(i -> i > 0).orElse(1), Optional.ofNullable(perPage).filter(i -> i > 0).orElse(defaultConsumerGroupsPageSize), search, Optional.ofNullable(orderBy).orElse(ConsumerGroupOrderingDTO.NAME), Optional.ofNullable(sortOrderDto).orElse(SortOrderDTO.ASC) ) .map(this::convertPage) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> resetConsumerGroupOffsets(String clusterName, String group, Mono resetDto, ServerWebExchange exchange) { return resetDto.flatMap(reset -> { var context = AccessContext.builder() .cluster(clusterName) .topic(reset.getTopic()) .topicActions(TopicAction.VIEW) .consumerGroupActions(RESET_OFFSETS) .operationName("resetConsumerGroupOffsets") .build(); Supplier> mono = () -> { var cluster = getCluster(clusterName); switch (reset.getResetType()) { case EARLIEST: return offsetsResetService .resetToEarliest(cluster, group, reset.getTopic(), reset.getPartitions()); case LATEST: return offsetsResetService .resetToLatest(cluster, group, reset.getTopic(), reset.getPartitions()); case TIMESTAMP: if (reset.getResetToTimestamp() == null) { return Mono.error( new ValidationException( "resetToTimestamp is required when TIMESTAMP reset type used" ) ); } return offsetsResetService .resetToTimestamp(cluster, group, reset.getTopic(), reset.getPartitions(), reset.getResetToTimestamp()); case OFFSET: if (CollectionUtils.isEmpty(reset.getPartitionsOffsets())) { return Mono.error( new ValidationException( "partitionsOffsets is required when OFFSET reset type used" ) ); } Map offsets = reset.getPartitionsOffsets().stream() .collect(toMap(PartitionOffsetDTO::getPartition, PartitionOffsetDTO::getOffset)); return offsetsResetService.resetToOffsets(cluster, group, reset.getTopic(), offsets); default: return Mono.error( new ValidationException("Unknown resetType " + reset.getResetType()) ); } }; return validateAccess(context) .then(mono.get()) .doOnEach(sig -> audit(context, sig)); }).thenReturn(ResponseEntity.ok().build()); } private ConsumerGroupsPageResponseDTO convertPage(ConsumerGroupService.ConsumerGroupsPage consumerGroupConsumerGroupsPage) { return new ConsumerGroupsPageResponseDTO() .pageCount(consumerGroupConsumerGroupsPage.totalPages()) .consumerGroups(consumerGroupConsumerGroupsPage.consumerGroups() .stream() .map(ConsumerGroupMapper::toDto) .toList()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KafkaConnectController.java ================================================ package com.provectus.kafka.ui.controller; import static com.provectus.kafka.ui.model.ConnectorActionDTO.RESTART; import static com.provectus.kafka.ui.model.ConnectorActionDTO.RESTART_ALL_TASKS; import static com.provectus.kafka.ui.model.ConnectorActionDTO.RESTART_FAILED_TASKS; import com.provectus.kafka.ui.api.KafkaConnectApi; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.ConnectorActionDTO; import com.provectus.kafka.ui.model.ConnectorColumnsToSortDTO; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigValidationResponseDTO; import com.provectus.kafka.ui.model.ConnectorPluginDTO; import com.provectus.kafka.ui.model.FullConnectorInfoDTO; import com.provectus.kafka.ui.model.NewConnectorDTO; import com.provectus.kafka.ui.model.SortOrderDTO; import com.provectus.kafka.ui.model.TaskDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.service.KafkaConnectService; import java.util.Comparator; import java.util.Map; import java.util.Set; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class KafkaConnectController extends AbstractController implements KafkaConnectApi { private static final Set RESTART_ACTIONS = Set.of(RESTART, RESTART_FAILED_TASKS, RESTART_ALL_TASKS); private static final String CONNECTOR_NAME = "connectorName"; private final KafkaConnectService kafkaConnectService; @Override public Mono>> getConnects(String clusterName, ServerWebExchange exchange) { Flux availableConnects = kafkaConnectService.getConnects(getCluster(clusterName)) .filterWhen(dto -> accessControlService.isConnectAccessible(dto, clusterName)); return Mono.just(ResponseEntity.ok(availableConnects)); } @Override public Mono>> getConnectors(String clusterName, String connectName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .operationName("getConnectors") .build(); return validateAccess(context) .thenReturn(ResponseEntity.ok(kafkaConnectService.getConnectorNames(getCluster(clusterName), connectName))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> createConnector(String clusterName, String connectName, @Valid Mono connector, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW, ConnectAction.CREATE) .operationName("createConnector") .build(); return validateAccess(context).then( kafkaConnectService.createConnector(getCluster(clusterName), connectName, connector) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getConnector(String clusterName, String connectName, String connectorName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .connector(connectorName) .operationName("getConnector") .build(); return validateAccess(context).then( kafkaConnectService.getConnector(getCluster(clusterName), connectName, connectorName) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> deleteConnector(String clusterName, String connectName, String connectorName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW, ConnectAction.EDIT) .operationName("deleteConnector") .operationParams(Map.of(CONNECTOR_NAME, connectName)) .build(); return validateAccess(context).then( kafkaConnectService.deleteConnector(getCluster(clusterName), connectName, connectorName) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getAllConnectors( String clusterName, String search, ConnectorColumnsToSortDTO orderBy, SortOrderDTO sortOrder, ServerWebExchange exchange ) { var context = AccessContext.builder() .cluster(clusterName) .connectActions(ConnectAction.VIEW, ConnectAction.EDIT) .operationName("getAllConnectors") .build(); var comparator = sortOrder == null || sortOrder.equals(SortOrderDTO.ASC) ? getConnectorsComparator(orderBy) : getConnectorsComparator(orderBy).reversed(); Flux job = kafkaConnectService.getAllConnectors(getCluster(clusterName), search) .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), clusterName)) .filterWhen(dto -> accessControlService.isConnectorAccessible(dto.getConnect(), dto.getName(), clusterName)) .sort(comparator); return Mono.just(ResponseEntity.ok(job)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getConnectorConfig(String clusterName, String connectName, String connectorName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .operationName("getConnectorConfig") .build(); return validateAccess(context).then( kafkaConnectService .getConnectorConfig(getCluster(clusterName), connectName, connectorName) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> setConnectorConfig(String clusterName, String connectName, String connectorName, Mono> requestBody, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW, ConnectAction.EDIT) .operationName("setConnectorConfig") .operationParams(Map.of(CONNECTOR_NAME, connectorName)) .build(); return validateAccess(context).then( kafkaConnectService .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) .map(ResponseEntity::ok)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateConnectorState(String clusterName, String connectName, String connectorName, ConnectorActionDTO action, ServerWebExchange exchange) { ConnectAction[] connectActions; if (RESTART_ACTIONS.contains(action)) { connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.RESTART}; } else { connectActions = new ConnectAction[] {ConnectAction.VIEW, ConnectAction.EDIT}; } var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(connectActions) .operationName("updateConnectorState") .operationParams(Map.of(CONNECTOR_NAME, connectorName)) .build(); return validateAccess(context).then( kafkaConnectService .updateConnectorState(getCluster(clusterName), connectName, connectorName, action) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getConnectorTasks(String clusterName, String connectName, String connectorName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .operationName("getConnectorTasks") .operationParams(Map.of(CONNECTOR_NAME, connectorName)) .build(); return validateAccess(context).thenReturn( ResponseEntity .ok(kafkaConnectService .getConnectorTasks(getCluster(clusterName), connectName, connectorName)) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> restartConnectorTask(String clusterName, String connectName, String connectorName, Integer taskId, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW, ConnectAction.RESTART) .operationName("restartConnectorTask") .operationParams(Map.of(CONNECTOR_NAME, connectorName)) .build(); return validateAccess(context).then( kafkaConnectService .restartConnectorTask(getCluster(clusterName), connectName, connectorName, taskId) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getConnectorPlugins( String clusterName, String connectName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .operationName("getConnectorPlugins") .build(); return validateAccess(context).then( Mono.just( ResponseEntity.ok( kafkaConnectService.getConnectorPlugins(getCluster(clusterName), connectName))) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> validateConnectorPluginConfig( String clusterName, String connectName, String pluginName, @Valid Mono> requestBody, ServerWebExchange exchange) { return kafkaConnectService .validateConnectorPluginConfig( getCluster(clusterName), connectName, pluginName, requestBody) .map(ResponseEntity::ok); } private Comparator getConnectorsComparator(ConnectorColumnsToSortDTO orderBy) { var defaultComparator = Comparator.comparing(FullConnectorInfoDTO::getName); if (orderBy == null) { return defaultComparator; } return switch (orderBy) { case CONNECT -> Comparator.comparing(FullConnectorInfoDTO::getConnect); case TYPE -> Comparator.comparing(FullConnectorInfoDTO::getType); case STATUS -> Comparator.comparing(fullConnectorInfoDTO -> fullConnectorInfoDTO.getStatus().getState()); default -> defaultComparator; }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KsqlController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.KsqlApi; import com.provectus.kafka.ui.model.KsqlCommandV2DTO; import com.provectus.kafka.ui.model.KsqlCommandV2ResponseDTO; import com.provectus.kafka.ui.model.KsqlResponseDTO; import com.provectus.kafka.ui.model.KsqlStreamDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableResponseDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.KsqlAction; import com.provectus.kafka.ui.service.ksql.KsqlServiceV2; import java.util.List; import java.util.Map; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class KsqlController extends AbstractController implements KsqlApi { private final KsqlServiceV2 ksqlServiceV2; @Override public Mono> executeKsql(String clusterName, Mono ksqlCmdDo, ServerWebExchange exchange) { return ksqlCmdDo.flatMap( command -> { var context = AccessContext.builder() .cluster(clusterName) .ksqlActions(KsqlAction.EXECUTE) .operationName("executeKsql") .operationParams(command) .build(); return validateAccess(context).thenReturn( new KsqlCommandV2ResponseDTO().pipeId( ksqlServiceV2.registerCommand( getCluster(clusterName), command.getKsql(), Optional.ofNullable(command.getStreamsProperties()).orElse(Map.of())))) .doOnEach(sig -> audit(context, sig)); } ) .map(ResponseEntity::ok); } @Override public Mono>> openKsqlResponsePipe(String clusterName, String pipeId, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .ksqlActions(KsqlAction.EXECUTE) .operationName("openKsqlResponsePipe") .build(); return validateAccess(context).thenReturn( ResponseEntity.ok(ksqlServiceV2.execute(pipeId) .map(table -> new KsqlResponseDTO() .table( new KsqlTableResponseDTO() .header(table.getHeader()) .columnNames(table.getColumnNames()) .values((List>) ((List) (table.getValues())))))) ); } @Override public Mono>> listStreams(String clusterName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .ksqlActions(KsqlAction.EXECUTE) .operationName("listStreams") .build(); return validateAccess(context) .thenReturn(ResponseEntity.ok(ksqlServiceV2.listStreams(getCluster(clusterName)))) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> listTables(String clusterName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .ksqlActions(KsqlAction.EXECUTE) .operationName("listTables") .build(); return validateAccess(context) .thenReturn(ResponseEntity.ok(ksqlServiceV2.listTables(getCluster(clusterName)))) .doOnEach(sig -> audit(context, sig)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/MessagesController.java ================================================ package com.provectus.kafka.ui.controller; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.MESSAGES_DELETE; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.MESSAGES_PRODUCE; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.MESSAGES_READ; import static com.provectus.kafka.ui.serde.api.Serde.Target.KEY; import static com.provectus.kafka.ui.serde.api.Serde.Target.VALUE; import static java.util.stream.Collectors.toMap; import com.provectus.kafka.ui.api.MessagesApi; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.CreateTopicMessageDTO; import com.provectus.kafka.ui.model.MessageFilterTypeDTO; import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.SeekTypeDTO; import com.provectus.kafka.ui.model.SerdeUsageDTO; import com.provectus.kafka.ui.model.SmartFilterTestExecutionDTO; import com.provectus.kafka.ui.model.SmartFilterTestExecutionResultDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.model.TopicSerdeSuggestionDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.AuditAction; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import com.provectus.kafka.ui.service.DeserializationService; import com.provectus.kafka.ui.service.MessagesService; import com.provectus.kafka.ui.util.DynamicConfigOperations; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.Nullable; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.common.TopicPartition; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @RestController @RequiredArgsConstructor @Slf4j public class MessagesController extends AbstractController implements MessagesApi { private final MessagesService messagesService; private final DeserializationService deserializationService; private final DynamicConfigOperations dynamicConfigOperations; @Override public Mono> deleteTopicMessages( String clusterName, String topicName, @Valid List partitions, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_DELETE) .build(); return validateAccess(context).>then( messagesService.deleteTopicMessages( getCluster(clusterName), topicName, Optional.ofNullable(partitions).orElse(List.of()) ).thenReturn(ResponseEntity.ok().build()) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> executeSmartFilterTest( Mono smartFilterTestExecutionDto, ServerWebExchange exchange) { return smartFilterTestExecutionDto .map(MessagesService::execSmartFilterTest) .map(ResponseEntity::ok); } @Override public Mono>> getTopicMessages(String clusterName, String topicName, SeekTypeDTO seekType, List seekTo, Integer limit, String q, MessageFilterTypeDTO filterQueryType, SeekDirectionDTO seekDirection, String keySerde, String valueSerde, ServerWebExchange exchange) { var contextBuilder = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_READ) .operationName("getTopicMessages"); if (StringUtils.isNoneEmpty(q) && MessageFilterTypeDTO.GROOVY_SCRIPT == filterQueryType) { dynamicConfigOperations.checkIfFilteringGroovyEnabled(); } if (auditService.isAuditTopic(getCluster(clusterName), topicName)) { contextBuilder.auditActions(AuditAction.VIEW); } seekType = seekType != null ? seekType : SeekTypeDTO.BEGINNING; seekDirection = seekDirection != null ? seekDirection : SeekDirectionDTO.FORWARD; filterQueryType = filterQueryType != null ? filterQueryType : MessageFilterTypeDTO.STRING_CONTAINS; var positions = new ConsumerPosition( seekType, topicName, parseSeekTo(topicName, seekType, seekTo) ); Mono>> job = Mono.just( ResponseEntity.ok( messagesService.loadMessages( getCluster(clusterName), topicName, positions, q, filterQueryType, limit, seekDirection, keySerde, valueSerde) ) ); var context = contextBuilder.build(); return validateAccess(context) .then(job) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> sendTopicMessages( String clusterName, String topicName, @Valid Mono createTopicMessage, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_PRODUCE) .operationName("sendTopicMessages") .build(); return validateAccess(context).then( createTopicMessage.flatMap(msg -> messagesService.sendMessage(getCluster(clusterName), topicName, msg).then() ).map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } /** * The format is [partition]::[offset] for specifying offsets * or [partition]::[timestamp in millis] for specifying timestamps. */ @Nullable private Map parseSeekTo(String topic, SeekTypeDTO seekType, List seekTo) { if (seekTo == null || seekTo.isEmpty()) { if (seekType == SeekTypeDTO.LATEST || seekType == SeekTypeDTO.BEGINNING) { return null; } throw new ValidationException("seekTo should be set if seekType is " + seekType); } return seekTo.stream() .map(p -> { String[] split = p.split("::"); if (split.length != 2) { throw new IllegalArgumentException( "Wrong seekTo argument format. See API docs for details"); } return Pair.of( new TopicPartition(topic, Integer.parseInt(split[0])), Long.parseLong(split[1]) ); }) .collect(toMap(Pair::getKey, Pair::getValue)); } @Override public Mono> getSerdes(String clusterName, String topicName, SerdeUsageDTO use, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(TopicAction.VIEW) .operationName("getSerdes") .build(); TopicSerdeSuggestionDTO dto = new TopicSerdeSuggestionDTO() .key(use == SerdeUsageDTO.SERIALIZE ? deserializationService.getSerdesForSerialize(getCluster(clusterName), topicName, KEY) : deserializationService.getSerdesForDeserialize(getCluster(clusterName), topicName, KEY)) .value(use == SerdeUsageDTO.SERIALIZE ? deserializationService.getSerdesForSerialize(getCluster(clusterName), topicName, VALUE) : deserializationService.getSerdesForDeserialize(getCluster(clusterName), topicName, VALUE)); return validateAccess(context).then( Mono.just(dto) .subscribeOn(Schedulers.boundedElastic()) .map(ResponseEntity::ok) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/SchemasController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.api.SchemasApi; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.mapper.KafkaSrMapper; import com.provectus.kafka.ui.mapper.KafkaSrMapperImpl; import com.provectus.kafka.ui.model.CompatibilityCheckResponseDTO; import com.provectus.kafka.ui.model.CompatibilityLevelDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.NewSchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaSubjectsResponseDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; import com.provectus.kafka.ui.service.SchemaRegistryService; import java.util.List; import java.util.Map; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class SchemasController extends AbstractController implements SchemasApi { private static final Integer DEFAULT_PAGE_SIZE = 25; private final KafkaSrMapper kafkaSrMapper = new KafkaSrMapperImpl(); private final SchemaRegistryService schemaRegistryService; @Override protected KafkaCluster getCluster(String clusterName) { var c = super.getCluster(clusterName); if (c.getSchemaRegistryClient() == null) { throw new ValidationException("Schema Registry is not set for cluster " + clusterName); } return c; } @Override public Mono> checkSchemaCompatibility( String clusterName, String subject, @Valid Mono newSchemaSubjectMono, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subject) .schemaActions(SchemaAction.VIEW) .operationName("checkSchemaCompatibility") .build(); return validateAccess(context).then( newSchemaSubjectMono.flatMap(subjectDTO -> schemaRegistryService.checksSchemaCompatibility( getCluster(clusterName), subject, kafkaSrMapper.fromDto(subjectDTO) )) .map(kafkaSrMapper::toDto) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> createNewSchema( String clusterName, @Valid Mono newSchemaSubjectMono, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schemaActions(SchemaAction.CREATE) .operationName("createNewSchema") .build(); return validateAccess(context).then( newSchemaSubjectMono.flatMap(newSubject -> schemaRegistryService.registerNewSchema( getCluster(clusterName), newSubject.getSubject(), kafkaSrMapper.fromDto(newSubject) ) ).map(kafkaSrMapper::toDto) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> deleteLatestSchema( String clusterName, String subject, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subject) .schemaActions(SchemaAction.DELETE) .operationName("deleteLatestSchema") .build(); return validateAccess(context).then( schemaRegistryService.deleteLatestSchemaSubject(getCluster(clusterName), subject) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } @Override public Mono> deleteSchema( String clusterName, String subject, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subject) .schemaActions(SchemaAction.DELETE) .operationName("deleteSchema") .build(); return validateAccess(context).then( schemaRegistryService.deleteSchemaSubjectEntirely(getCluster(clusterName), subject) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } @Override public Mono> deleteSchemaByVersion( String clusterName, String subjectName, Integer version, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subjectName) .schemaActions(SchemaAction.DELETE) .operationName("deleteSchemaByVersion") .build(); return validateAccess(context).then( schemaRegistryService.deleteSchemaSubjectByVersion(getCluster(clusterName), subjectName, version) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } @Override public Mono>> getAllVersionsBySubject( String clusterName, String subjectName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subjectName) .schemaActions(SchemaAction.VIEW) .operationName("getAllVersionsBySubject") .build(); Flux schemas = schemaRegistryService.getAllVersionsBySubject(getCluster(clusterName), subjectName) .map(kafkaSrMapper::toDto); return validateAccess(context) .thenReturn(ResponseEntity.ok(schemas)) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> getGlobalSchemaCompatibilityLevel( String clusterName, ServerWebExchange exchange) { return schemaRegistryService.getGlobalSchemaCompatibilityLevel(getCluster(clusterName)) .map(c -> new CompatibilityLevelDTO().compatibility(kafkaSrMapper.toDto(c))) .map(ResponseEntity::ok) .defaultIfEmpty(ResponseEntity.notFound().build()); } @Override public Mono> getLatestSchema(String clusterName, String subject, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subject) .schemaActions(SchemaAction.VIEW) .operationName("getLatestSchema") .build(); return validateAccess(context).then( schemaRegistryService.getLatestSchemaVersionBySubject(getCluster(clusterName), subject) .map(kafkaSrMapper::toDto) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getSchemaByVersion( String clusterName, String subject, Integer version, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schema(subject) .schemaActions(SchemaAction.VIEW) .operationName("getSchemaByVersion") .operationParams(Map.of("subject", subject, "version", version)) .build(); return validateAccess(context).then( schemaRegistryService.getSchemaSubjectByVersion( getCluster(clusterName), subject, version) .map(kafkaSrMapper::toDto) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getSchemas(String clusterName, @Valid Integer pageNum, @Valid Integer perPage, @Valid String search, ServerWebExchange serverWebExchange) { var context = AccessContext.builder() .cluster(clusterName) .operationName("getSchemas") .build(); return schemaRegistryService .getAllSubjectNames(getCluster(clusterName)) .flatMapIterable(l -> l) .filterWhen(schema -> accessControlService.isSchemaAccessible(schema, clusterName)) .collectList() .flatMap(subjects -> { int pageSize = perPage != null && perPage > 0 ? perPage : DEFAULT_PAGE_SIZE; int subjectToSkip = ((pageNum != null && pageNum > 0 ? pageNum : 1) - 1) * pageSize; List filteredSubjects = subjects .stream() .filter(subj -> search == null || StringUtils.containsIgnoreCase(subj, search)) .sorted().toList(); var totalPages = (filteredSubjects.size() / pageSize) + (filteredSubjects.size() % pageSize == 0 ? 0 : 1); List subjectsToRender = filteredSubjects.stream() .skip(subjectToSkip) .limit(pageSize) .toList(); return schemaRegistryService.getAllLatestVersionSchemas(getCluster(clusterName), subjectsToRender) .map(subjs -> subjs.stream().map(kafkaSrMapper::toDto).toList()) .map(subjs -> new SchemaSubjectsResponseDTO().pageCount(totalPages).schemas(subjs)); }).map(ResponseEntity::ok) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateGlobalSchemaCompatibilityLevel( String clusterName, @Valid Mono compatibilityLevelMono, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schemaActions(SchemaAction.MODIFY_GLOBAL_COMPATIBILITY) .operationName("updateGlobalSchemaCompatibilityLevel") .build(); return validateAccess(context).then( compatibilityLevelMono .flatMap(compatibilityLevelDTO -> schemaRegistryService.updateGlobalSchemaCompatibility( getCluster(clusterName), kafkaSrMapper.fromDto(compatibilityLevelDTO.getCompatibility()) )) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } @Override public Mono> updateSchemaCompatibilityLevel( String clusterName, String subject, @Valid Mono compatibilityLevelMono, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .schemaActions(SchemaAction.EDIT) .operationName("updateSchemaCompatibilityLevel") .operationParams(Map.of("subject", subject)) .build(); return validateAccess(context).then( compatibilityLevelMono .flatMap(compatibilityLevelDTO -> schemaRegistryService.updateSchemaCompatibility( getCluster(clusterName), subject, kafkaSrMapper.fromDto(compatibilityLevelDTO.getCompatibility()) )) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/StaticController.java ================================================ package com.provectus.kafka.ui.controller; import com.provectus.kafka.ui.util.ResourceUtil; import java.util.concurrent.atomic.AtomicReference; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class StaticController { @Value("classpath:static/index.html") private Resource indexFile; @Value("classpath:static/manifest.json") private Resource manifestFile; private final AtomicReference renderedIndexFile = new AtomicReference<>(); private final AtomicReference renderedManifestFile = new AtomicReference<>(); @GetMapping(value = "/index.html", produces = {"text/html"}) public Mono> getIndex(ServerWebExchange exchange) { return Mono.just(ResponseEntity.ok(getRenderedFile(exchange, renderedIndexFile, indexFile))); } @GetMapping(value = "/manifest.json", produces = {"application/json"}) public Mono> getManifest(ServerWebExchange exchange) { return Mono.just(ResponseEntity.ok(getRenderedFile(exchange, renderedManifestFile, manifestFile))); } public String getRenderedFile(ServerWebExchange exchange, AtomicReference renderedFile, Resource file) { String rendered = renderedFile.get(); if (rendered == null) { rendered = buildFile(file, exchange.getRequest().getPath().contextPath().value()); if (renderedFile.compareAndSet(null, rendered)) { return rendered; } else { return renderedFile.get(); } } else { return rendered; } } @SneakyThrows private String buildFile(Resource file, String contextPath) { return ResourceUtil.readAsString(file) .replace("\"assets/", "\"" + contextPath + "/assets/") .replace("PUBLIC-PATH-VARIABLE", contextPath); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/TopicsController.java ================================================ package com.provectus.kafka.ui.controller; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.CREATE; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.DELETE; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.EDIT; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.MESSAGES_READ; import static com.provectus.kafka.ui.model.rbac.permission.TopicAction.VIEW; import static java.util.stream.Collectors.toList; import com.provectus.kafka.ui.api.TopicsApi; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.InternalTopicConfig; import com.provectus.kafka.ui.model.PartitionsIncreaseDTO; import com.provectus.kafka.ui.model.PartitionsIncreaseResponseDTO; import com.provectus.kafka.ui.model.ReplicationFactorChangeDTO; import com.provectus.kafka.ui.model.ReplicationFactorChangeResponseDTO; import com.provectus.kafka.ui.model.SortOrderDTO; import com.provectus.kafka.ui.model.TopicAnalysisDTO; import com.provectus.kafka.ui.model.TopicColumnsToSortDTO; import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; import com.provectus.kafka.ui.model.TopicProducerStateDTO; import com.provectus.kafka.ui.model.TopicUpdateDTO; import com.provectus.kafka.ui.model.TopicsResponseDTO; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.TopicsService; import com.provectus.kafka.ui.service.analyze.TopicAnalysisService; import java.util.Comparator; import java.util.List; import java.util.Map; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @RestController @RequiredArgsConstructor @Slf4j public class TopicsController extends AbstractController implements TopicsApi { private static final Integer DEFAULT_PAGE_SIZE = 25; private final TopicsService topicsService; private final TopicAnalysisService topicAnalysisService; private final ClusterMapper clusterMapper; @Override public Mono> createTopic( String clusterName, @Valid Mono topicCreationMono, ServerWebExchange exchange) { return topicCreationMono.flatMap(topicCreation -> { var context = AccessContext.builder() .cluster(clusterName) .topicActions(CREATE) .operationName("createTopic") .operationParams(topicCreation) .build(); return validateAccess(context) .then(topicsService.createTopic(getCluster(clusterName), topicCreation)) .map(clusterMapper::toTopic) .map(s -> new ResponseEntity<>(s, HttpStatus.OK)) .switchIfEmpty(Mono.just(ResponseEntity.notFound().build())) .doOnEach(sig -> audit(context, sig)); }); } @Override public Mono> recreateTopic(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW, CREATE, DELETE) .operationName("recreateTopic") .build(); return validateAccess(context).then( topicsService.recreateTopic(getCluster(clusterName), topicName) .map(clusterMapper::toTopic) .map(s -> new ResponseEntity<>(s, HttpStatus.CREATED)) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> cloneTopic( String clusterName, String topicName, String newTopicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW, CREATE) .operationName("cloneTopic") .operationParams(Map.of("newTopicName", newTopicName)) .build(); return validateAccess(context) .then(topicsService.cloneTopic(getCluster(clusterName), topicName, newTopicName) .map(clusterMapper::toTopic) .map(s -> new ResponseEntity<>(s, HttpStatus.CREATED)) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> deleteTopic( String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(DELETE) .operationName("deleteTopic") .build(); return validateAccess(context) .then( topicsService.deleteTopic(getCluster(clusterName), topicName) .thenReturn(ResponseEntity.ok().build()) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getTopicConfigs( String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW) .operationName("getTopicConfigs") .build(); return validateAccess(context).then( topicsService.getTopicConfigs(getCluster(clusterName), topicName) .map(lst -> lst.stream() .map(InternalTopicConfig::from) .map(clusterMapper::toTopicConfig) .toList()) .map(Flux::fromIterable) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getTopicDetails( String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW) .operationName("getTopicDetails") .build(); return validateAccess(context).then( topicsService.getTopicDetails(getCluster(clusterName), topicName) .map(clusterMapper::toTopicDetails) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> getTopics(String clusterName, @Valid Integer page, @Valid Integer perPage, @Valid Boolean showInternal, @Valid String search, @Valid TopicColumnsToSortDTO orderBy, @Valid SortOrderDTO sortOrder, ServerWebExchange exchange) { AccessContext context = AccessContext.builder() .cluster(clusterName) .operationName("getTopics") .build(); return topicsService.getTopicsForPagination(getCluster(clusterName)) .flatMap(topics -> accessControlService.filterViewableTopics(topics, clusterName)) .flatMap(topics -> { int pageSize = perPage != null && perPage > 0 ? perPage : DEFAULT_PAGE_SIZE; var topicsToSkip = ((page != null && page > 0 ? page : 1) - 1) * pageSize; var comparator = sortOrder == null || !sortOrder.equals(SortOrderDTO.DESC) ? getComparatorForTopic(orderBy) : getComparatorForTopic(orderBy).reversed(); List filtered = topics.stream() .filter(topic -> !topic.isInternal() || showInternal != null && showInternal) .filter(topic -> search == null || StringUtils.containsIgnoreCase(topic.getName(), search)) .sorted(comparator) .toList(); var totalPages = (filtered.size() / pageSize) + (filtered.size() % pageSize == 0 ? 0 : 1); List topicsPage = filtered.stream() .skip(topicsToSkip) .limit(pageSize) .map(InternalTopic::getName) .collect(toList()); return topicsService.loadTopics(getCluster(clusterName), topicsPage) .map(topicsToRender -> new TopicsResponseDTO() .topics(topicsToRender.stream().map(clusterMapper::toTopic).toList()) .pageCount(totalPages)); }) .map(ResponseEntity::ok) .doOnEach(sig -> audit(context, sig)); } @Override public Mono> updateTopic( String clusterName, String topicName, @Valid Mono topicUpdate, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW, EDIT) .operationName("updateTopic") .build(); return validateAccess(context).then( topicsService .updateTopic(getCluster(clusterName), topicName, topicUpdate) .map(clusterMapper::toTopic) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> increaseTopicPartitions( String clusterName, String topicName, Mono partitionsIncrease, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW, EDIT) .build(); return validateAccess(context).then( partitionsIncrease.flatMap(partitions -> topicsService.increaseTopicPartitions(getCluster(clusterName), topicName, partitions) ).map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> changeReplicationFactor( String clusterName, String topicName, Mono replicationFactorChange, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW, EDIT) .operationName("changeReplicationFactor") .build(); return validateAccess(context).then( replicationFactorChange .flatMap(rfc -> topicsService.changeReplicationFactor(getCluster(clusterName), topicName, rfc)) .map(ResponseEntity::ok) ).doOnEach(sig -> audit(context, sig)); } @Override public Mono> analyzeTopic(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_READ) .operationName("analyzeTopic") .build(); return validateAccess(context).then( topicAnalysisService.analyze(getCluster(clusterName), topicName) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()) ); } @Override public Mono> cancelTopicAnalysis(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_READ) .operationName("cancelTopicAnalysis") .build(); return validateAccess(context) .then(Mono.fromRunnable(() -> topicAnalysisService.cancelAnalysis(getCluster(clusterName), topicName))) .doOnEach(sig -> audit(context, sig)) .thenReturn(ResponseEntity.ok().build()); } @Override public Mono> getTopicAnalysis(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(MESSAGES_READ) .operationName("getTopicAnalysis") .build(); return validateAccess(context) .thenReturn(topicAnalysisService.getTopicAnalysis(getCluster(clusterName), topicName) .map(ResponseEntity::ok) .orElseGet(() -> ResponseEntity.notFound().build())) .doOnEach(sig -> audit(context, sig)); } @Override public Mono>> getActiveProducerStates(String clusterName, String topicName, ServerWebExchange exchange) { var context = AccessContext.builder() .cluster(clusterName) .topic(topicName) .topicActions(VIEW) .operationName("getActiveProducerStates") .build(); Comparator ordering = Comparator.comparingInt(TopicProducerStateDTO::getPartition) .thenComparing(Comparator.comparing(TopicProducerStateDTO::getProducerId).reversed()); Flux states = topicsService.getActiveProducersState(getCluster(clusterName), topicName) .flatMapMany(statesMap -> Flux.fromStream( statesMap.entrySet().stream() .flatMap(e -> e.getValue().stream().map(p -> clusterMapper.map(e.getKey().partition(), p))) .sorted(ordering))); return validateAccess(context) .thenReturn(states) .map(ResponseEntity::ok) .doOnEach(sig -> audit(context, sig)); } private Comparator getComparatorForTopic( TopicColumnsToSortDTO orderBy) { var defaultComparator = Comparator.comparing(InternalTopic::getName); if (orderBy == null) { return defaultComparator; } switch (orderBy) { case TOTAL_PARTITIONS: return Comparator.comparing(InternalTopic::getPartitionCount); case OUT_OF_SYNC_REPLICAS: return Comparator.comparing(t -> t.getReplicas() - t.getInSyncReplicas()); case REPLICATION_FACTOR: return Comparator.comparing(InternalTopic::getReplicationFactor); case SIZE: return Comparator.comparing(InternalTopic::getSegmentSize); case NAME: default: return defaultComparator; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/AbstractEmitter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; import reactor.core.publisher.FluxSink; abstract class AbstractEmitter implements java.util.function.Consumer> { private final MessagesProcessing messagesProcessing; private final PollingSettings pollingSettings; protected AbstractEmitter(MessagesProcessing messagesProcessing, PollingSettings pollingSettings) { this.messagesProcessing = messagesProcessing; this.pollingSettings = pollingSettings; } protected PolledRecords poll(FluxSink sink, EnhancedConsumer consumer) { var records = consumer.pollEnhanced(pollingSettings.getPollTimeout()); sendConsuming(sink, records); return records; } protected boolean sendLimitReached() { return messagesProcessing.limitReached(); } protected void send(FluxSink sink, Iterable> records) { messagesProcessing.send(sink, records); } protected void sendPhase(FluxSink sink, String name) { messagesProcessing.sendPhase(sink, name); } protected void sendConsuming(FluxSink sink, PolledRecords records) { messagesProcessing.sentConsumingInfo(sink, records); } protected void sendFinishStatsAndCompleteSink(FluxSink sink) { messagesProcessing.sendFinishEvent(sink); sink.complete(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/BackwardEmitter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import java.util.Comparator; import java.util.Map; import java.util.TreeMap; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.kafka.common.TopicPartition; public class BackwardEmitter extends RangePollingEmitter { public BackwardEmitter(Supplier consumerSupplier, ConsumerPosition consumerPosition, int messagesPerPage, ConsumerRecordDeserializer deserializer, Predicate filter, PollingSettings pollingSettings) { super( consumerSupplier, consumerPosition, messagesPerPage, new MessagesProcessing( deserializer, filter, false, messagesPerPage ), pollingSettings ); } @Override protected TreeMap nextPollingRange(TreeMap prevRange, SeekOperations seekOperations) { TreeMap readToOffsets = new TreeMap<>(Comparator.comparingInt(TopicPartition::partition)); if (prevRange.isEmpty()) { readToOffsets.putAll(seekOperations.getOffsetsForSeek()); } else { readToOffsets.putAll( prevRange.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().from())) ); } int msgsToPollPerPartition = (int) Math.ceil((double) messagesPerPage / readToOffsets.size()); TreeMap result = new TreeMap<>(Comparator.comparingInt(TopicPartition::partition)); readToOffsets.forEach((tp, toOffset) -> { long tpStartOffset = seekOperations.getBeginOffsets().get(tp); if (toOffset > tpStartOffset) { result.put(tp, new FromToOffset(Math.max(tpStartOffset, toOffset - msgsToPollPerPartition), toOffset)); } }); return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ConsumingStats.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.TopicMessageConsumingDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import reactor.core.publisher.FluxSink; class ConsumingStats { private long bytes = 0; private int records = 0; private long elapsed = 0; private int filterApplyErrors = 0; void sendConsumingEvt(FluxSink sink, PolledRecords polledRecords) { bytes += polledRecords.bytes(); records += polledRecords.count(); elapsed += polledRecords.elapsed().toMillis(); sink.next( new TopicMessageEventDTO() .type(TopicMessageEventDTO.TypeEnum.CONSUMING) .consuming(createConsumingStats()) ); } void incFilterApplyError() { filterApplyErrors++; } void sendFinishEvent(FluxSink sink) { sink.next( new TopicMessageEventDTO() .type(TopicMessageEventDTO.TypeEnum.DONE) .consuming(createConsumingStats()) ); } private TopicMessageConsumingDTO createConsumingStats() { return new TopicMessageConsumingDTO() .bytesConsumed(bytes) .elapsedMs(elapsed) .isCancelled(false) .filterApplyErrors(filterApplyErrors) .messagesConsumed(records); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/EnhancedConsumer.java ================================================ package com.provectus.kafka.ui.emitter; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.provectus.kafka.ui.util.ApplicationMetrics; import java.time.Duration; import java.util.Collection; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.experimental.Delegate; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.BytesDeserializer; import org.apache.kafka.common.utils.Bytes; public class EnhancedConsumer extends KafkaConsumer { private final PollingThrottler throttler; private final ApplicationMetrics metrics; private String pollingTopic; public EnhancedConsumer(Properties properties, PollingThrottler throttler, ApplicationMetrics metrics) { super(properties, new BytesDeserializer(), new BytesDeserializer()); this.throttler = throttler; this.metrics = metrics; metrics.activeConsumers().incrementAndGet(); } public PolledRecords pollEnhanced(Duration dur) { var stopwatch = Stopwatch.createStarted(); ConsumerRecords polled = poll(dur); PolledRecords polledEnhanced = PolledRecords.create(polled, stopwatch.elapsed()); var throttled = throttler.throttleAfterPoll(polledEnhanced.bytes()); metrics.meterPolledRecords(pollingTopic, polledEnhanced, throttled); return polledEnhanced; } @Override public void assign(Collection partitions) { super.assign(partitions); Set assignedTopics = partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet()); Preconditions.checkState(assignedTopics.size() == 1); this.pollingTopic = assignedTopics.iterator().next(); } @Override public void subscribe(Pattern pattern) { throw new UnsupportedOperationException(); } @Override public void subscribe(Collection topics) { throw new UnsupportedOperationException(); } @Override public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { throw new UnsupportedOperationException(); } @Override public void subscribe(Collection topics, ConsumerRebalanceListener listener) { throw new UnsupportedOperationException(); } @Override public void close(Duration timeout) { metrics.activeConsumers().decrementAndGet(); super.close(timeout); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ForwardEmitter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import java.util.Comparator; import java.util.Map; import java.util.TreeMap; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.kafka.common.TopicPartition; public class ForwardEmitter extends RangePollingEmitter { public ForwardEmitter(Supplier consumerSupplier, ConsumerPosition consumerPosition, int messagesPerPage, ConsumerRecordDeserializer deserializer, Predicate filter, PollingSettings pollingSettings) { super( consumerSupplier, consumerPosition, messagesPerPage, new MessagesProcessing( deserializer, filter, true, messagesPerPage ), pollingSettings ); } @Override protected TreeMap nextPollingRange(TreeMap prevRange, SeekOperations seekOperations) { TreeMap readFromOffsets = new TreeMap<>(Comparator.comparingInt(TopicPartition::partition)); if (prevRange.isEmpty()) { readFromOffsets.putAll(seekOperations.getOffsetsForSeek()); } else { readFromOffsets.putAll( prevRange.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().to())) ); } int msgsToPollPerPartition = (int) Math.ceil((double) messagesPerPage / readFromOffsets.size()); TreeMap result = new TreeMap<>(Comparator.comparingInt(TopicPartition::partition)); readFromOffsets.forEach((tp, fromOffset) -> { long tpEndOffset = seekOperations.getEndOffsets().get(tp); if (fromOffset < tpEndOffset) { result.put(tp, new FromToOffset(fromOffset, Math.min(tpEndOffset, fromOffset + msgsToPollPerPartition))); } }); return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/MessageFilters.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.MessageFilterTypeDTO; import com.provectus.kafka.ui.model.TopicMessageDTO; import groovy.json.JsonSlurper; import java.util.function.Predicate; import javax.annotation.Nullable; import javax.script.CompiledScript; import javax.script.ScriptEngineManager; import javax.script.ScriptException; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.codehaus.groovy.jsr223.GroovyScriptEngineImpl; @Slf4j public class MessageFilters { private static GroovyScriptEngineImpl GROOVY_ENGINE; private MessageFilters() { } public static Predicate createMsgFilter(String query, MessageFilterTypeDTO type) { switch (type) { case STRING_CONTAINS: return containsStringFilter(query); case GROOVY_SCRIPT: return groovyScriptFilter(query); default: throw new IllegalStateException("Unknown query type: " + type); } } static Predicate containsStringFilter(String string) { return msg -> StringUtils.contains(msg.getKey(), string) || StringUtils.contains(msg.getContent(), string); } static Predicate groovyScriptFilter(String script) { var engine = getGroovyEngine(); var compiledScript = compileScript(engine, script); var jsonSlurper = new JsonSlurper(); return new Predicate() { @SneakyThrows @Override public boolean test(TopicMessageDTO msg) { var bindings = engine.createBindings(); bindings.put("partition", msg.getPartition()); bindings.put("offset", msg.getOffset()); bindings.put("timestampMs", msg.getTimestamp().toInstant().toEpochMilli()); bindings.put("keyAsText", msg.getKey()); bindings.put("valueAsText", msg.getContent()); bindings.put("headers", msg.getHeaders()); bindings.put("key", parseToJsonOrReturnAsIs(jsonSlurper, msg.getKey())); bindings.put("value", parseToJsonOrReturnAsIs(jsonSlurper, msg.getContent())); var result = compiledScript.eval(bindings); if (result instanceof Boolean) { return (Boolean) result; } else { throw new ValidationException( "Unexpected script result: %s, Boolean should be returned instead".formatted(result)); } } }; } @Nullable private static Object parseToJsonOrReturnAsIs(JsonSlurper parser, @Nullable String str) { if (str == null) { return null; } try { return parser.parseText(str); } catch (Exception e) { return str; } } private static synchronized GroovyScriptEngineImpl getGroovyEngine() { // it is pretty heavy object, so initializing it on-demand if (GROOVY_ENGINE == null) { GROOVY_ENGINE = (GroovyScriptEngineImpl) new ScriptEngineManager().getEngineByName("groovy"); } return GROOVY_ENGINE; } private static CompiledScript compileScript(GroovyScriptEngineImpl engine, String script) { try { return engine.compile(script); } catch (ScriptException e) { throw new ValidationException("Script syntax error: " + e.getMessage()); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/MessagesProcessing.java ================================================ package com.provectus.kafka.ui.emitter; import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.model.TopicMessagePhaseDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.function.Predicate; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; import reactor.core.publisher.FluxSink; @Slf4j @RequiredArgsConstructor class MessagesProcessing { private final ConsumingStats consumingStats = new ConsumingStats(); private long sentMessages = 0; private final ConsumerRecordDeserializer deserializer; private final Predicate filter; private final boolean ascendingSortBeforeSend; private final @Nullable Integer limit; boolean limitReached() { return limit != null && sentMessages >= limit; } void send(FluxSink sink, Iterable> polled) { sortForSending(polled, ascendingSortBeforeSend) .forEach(rec -> { if (!limitReached() && !sink.isCancelled()) { TopicMessageDTO topicMessage = deserializer.deserialize(rec); try { if (filter.test(topicMessage)) { sink.next( new TopicMessageEventDTO() .type(TopicMessageEventDTO.TypeEnum.MESSAGE) .message(topicMessage) ); sentMessages++; } } catch (Exception e) { consumingStats.incFilterApplyError(); log.trace("Error applying filter for message {}", topicMessage); } } }); } void sentConsumingInfo(FluxSink sink, PolledRecords polledRecords) { if (!sink.isCancelled()) { consumingStats.sendConsumingEvt(sink, polledRecords); } } void sendFinishEvent(FluxSink sink) { if (!sink.isCancelled()) { consumingStats.sendFinishEvent(sink); } } void sendPhase(FluxSink sink, String name) { if (!sink.isCancelled()) { sink.next( new TopicMessageEventDTO() .type(TopicMessageEventDTO.TypeEnum.PHASE) .phase(new TopicMessagePhaseDTO().name(name)) ); } } /* * Sorting by timestamps, BUT requesting that records within same partitions should be ordered by offsets. */ @VisibleForTesting static Iterable> sortForSending(Iterable> records, boolean asc) { Comparator offsetComparator = asc ? Comparator.comparingLong(ConsumerRecord::offset) : Comparator.comparingLong(ConsumerRecord::offset).reversed(); // partition -> sorted by offsets records Map>> perPartition = Streams.stream(records) .collect( groupingBy( ConsumerRecord::partition, TreeMap::new, collectingAndThen(toList(), lst -> lst.stream().sorted(offsetComparator).toList()))); Comparator tsComparator = asc ? Comparator.comparing(ConsumerRecord::timestamp) : Comparator.comparingLong(ConsumerRecord::timestamp).reversed(); // merge-sorting records from partitions one by one using timestamp comparator return Iterables.mergeSorted(perPartition.values(), tsComparator); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/OffsetsInfo.java ================================================ package com.provectus.kafka.ui.emitter; import com.google.common.base.Preconditions; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.TopicPartition; @Slf4j @Getter class OffsetsInfo { private final Consumer consumer; private final Map beginOffsets; private final Map endOffsets; private final Set nonEmptyPartitions = new HashSet<>(); private final Set emptyPartitions = new HashSet<>(); OffsetsInfo(Consumer consumer, String topic) { this(consumer, consumer.partitionsFor(topic).stream() .map(pi -> new TopicPartition(topic, pi.partition())) .toList() ); } OffsetsInfo(Consumer consumer, Collection targetPartitions) { this.consumer = consumer; this.beginOffsets = consumer.beginningOffsets(targetPartitions); this.endOffsets = consumer.endOffsets(targetPartitions); endOffsets.forEach((tp, endOffset) -> { var beginningOffset = beginOffsets.get(tp); if (endOffset > beginningOffset) { nonEmptyPartitions.add(tp); } else { emptyPartitions.add(tp); } }); } boolean assignedPartitionsFullyPolled() { for (var tp : consumer.assignment()) { Preconditions.checkArgument(endOffsets.containsKey(tp)); if (endOffsets.get(tp) > consumer.position(tp)) { return false; } } return true; } long summaryOffsetsRange() { MutableLong cnt = new MutableLong(); nonEmptyPartitions.forEach(tp -> cnt.add(endOffsets.get(tp) - beginOffsets.get(tp))); return cnt.getValue(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/PolledRecords.java ================================================ package com.provectus.kafka.ui.emitter; import java.time.Duration; import java.util.Iterator; import java.util.List; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.utils.Bytes; public record PolledRecords(int count, int bytes, Duration elapsed, ConsumerRecords records) implements Iterable> { static PolledRecords create(ConsumerRecords polled, Duration pollDuration) { return new PolledRecords( polled.count(), calculatePolledRecSize(polled), pollDuration, polled ); } public List> records(TopicPartition tp) { return records.records(tp); } @Override public Iterator> iterator() { return records.iterator(); } private static int calculatePolledRecSize(Iterable> recs) { int polledBytes = 0; for (ConsumerRecord rec : recs) { for (Header header : rec.headers()) { polledBytes += (header.key() != null ? header.key().getBytes().length : 0) + (header.value() != null ? header.value().length : 0); } polledBytes += rec.key() == null ? 0 : rec.serializedKeySize(); polledBytes += rec.value() == null ? 0 : rec.serializedValueSize(); } return polledBytes; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/PollingSettings.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.config.ClustersProperties; import java.time.Duration; import java.util.Optional; import java.util.function.Supplier; public class PollingSettings { private static final Duration DEFAULT_POLL_TIMEOUT = Duration.ofMillis(1_000); private final Duration pollTimeout; private final Supplier throttlerSupplier; public static PollingSettings create(ClustersProperties.Cluster cluster, ClustersProperties clustersProperties) { var pollingProps = Optional.ofNullable(clustersProperties.getPolling()) .orElseGet(ClustersProperties.PollingProperties::new); var pollTimeout = pollingProps.getPollTimeoutMs() != null ? Duration.ofMillis(pollingProps.getPollTimeoutMs()) : DEFAULT_POLL_TIMEOUT; return new PollingSettings( pollTimeout, PollingThrottler.throttlerSupplier(cluster) ); } public static PollingSettings createDefault() { return new PollingSettings( DEFAULT_POLL_TIMEOUT, PollingThrottler::noop ); } private PollingSettings(Duration pollTimeout, Supplier throttlerSupplier) { this.pollTimeout = pollTimeout; this.throttlerSupplier = throttlerSupplier; } public Duration getPollTimeout() { return pollTimeout; } public PollingThrottler getPollingThrottler() { return throttlerSupplier.get(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/PollingThrottler.java ================================================ package com.provectus.kafka.ui.emitter; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.RateLimiter; import com.provectus.kafka.ui.config.ClustersProperties; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; @Slf4j public class PollingThrottler { public static Supplier throttlerSupplier(ClustersProperties.Cluster cluster) { Long rate = cluster.getPollingThrottleRate(); if (rate == null || rate <= 0) { return PollingThrottler::noop; } // RateLimiter instance should be shared across all created throttlers var rateLimiter = RateLimiter.create(rate); return () -> new PollingThrottler(cluster.getName(), rateLimiter); } private final String clusterName; private final RateLimiter rateLimiter; private boolean throttled; @VisibleForTesting public PollingThrottler(String clusterName, RateLimiter rateLimiter) { this.clusterName = clusterName; this.rateLimiter = rateLimiter; } public static PollingThrottler noop() { return new PollingThrottler("noop", RateLimiter.create(Long.MAX_VALUE)); } //returns true if polling was throttled public boolean throttleAfterPoll(int polledBytes) { if (polledBytes > 0) { double sleptSeconds = rateLimiter.acquire(polledBytes); if (!throttled && sleptSeconds > 0.0) { throttled = true; log.debug("Polling throttling enabled for cluster {} at rate {} bytes/sec", clusterName, rateLimiter.getRate()); return true; } } return false; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/RangePollingEmitter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import java.util.ArrayList; import java.util.List; import java.util.TreeMap; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.utils.Bytes; import reactor.core.publisher.FluxSink; @Slf4j abstract class RangePollingEmitter extends AbstractEmitter { private final Supplier consumerSupplier; protected final ConsumerPosition consumerPosition; protected final int messagesPerPage; protected RangePollingEmitter(Supplier consumerSupplier, ConsumerPosition consumerPosition, int messagesPerPage, MessagesProcessing messagesProcessing, PollingSettings pollingSettings) { super(messagesProcessing, pollingSettings); this.consumerPosition = consumerPosition; this.messagesPerPage = messagesPerPage; this.consumerSupplier = consumerSupplier; } protected record FromToOffset(/*inclusive*/ long from, /*exclusive*/ long to) { } //should return empty map if polling should be stopped protected abstract TreeMap nextPollingRange( TreeMap prevRange, //empty on start SeekOperations seekOperations ); @Override public void accept(FluxSink sink) { log.debug("Starting polling for {}", consumerPosition); try (EnhancedConsumer consumer = consumerSupplier.get()) { sendPhase(sink, "Consumer created"); var seekOperations = SeekOperations.create(consumer, consumerPosition); TreeMap pollRange = nextPollingRange(new TreeMap<>(), seekOperations); log.debug("Starting from offsets {}", pollRange); while (!sink.isCancelled() && !pollRange.isEmpty() && !sendLimitReached()) { var polled = poll(consumer, sink, pollRange); send(sink, polled); pollRange = nextPollingRange(pollRange, seekOperations); } if (sink.isCancelled()) { log.debug("Polling finished due to sink cancellation"); } sendFinishStatsAndCompleteSink(sink); log.debug("Polling finished"); } catch (InterruptException kafkaInterruptException) { log.debug("Polling finished due to thread interruption"); sink.complete(); } catch (Exception e) { log.error("Error occurred while consuming records", e); sink.error(e); } } private List> poll(EnhancedConsumer consumer, FluxSink sink, TreeMap range) { log.trace("Polling range {}", range); sendPhase(sink, "Polling partitions: %s".formatted(range.keySet().stream().map(TopicPartition::partition).sorted().toList())); consumer.assign(range.keySet()); range.forEach((tp, fromTo) -> consumer.seek(tp, fromTo.from)); List> result = new ArrayList<>(); while (!sink.isCancelled() && consumer.paused().size() < range.size()) { var polledRecords = poll(sink, consumer); range.forEach((tp, fromTo) -> { polledRecords.records(tp).stream() .filter(r -> r.offset() < fromTo.to) .forEach(result::add); //next position is out of target range -> pausing partition if (consumer.position(tp) >= fromTo.to) { consumer.pause(List.of(tp)); } }); } consumer.resume(consumer.paused()); return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/ResultSizeLimiter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; public class ResultSizeLimiter implements Predicate { private final AtomicInteger processed = new AtomicInteger(); private final int limit; public ResultSizeLimiter(int limit) { this.limit = limit; } @Override public boolean test(TopicMessageEventDTO event) { if (event.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) { final int i = processed.incrementAndGet(); return i <= limit; } return true; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/SeekOperations.java ================================================ package com.provectus.kafka.ui.emitter; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.SeekTypeDTO; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.common.TopicPartition; @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public class SeekOperations { private final Consumer consumer; private final OffsetsInfo offsetsInfo; private final Map offsetsForSeek; //only contains non-empty partitions! public static SeekOperations create(Consumer consumer, ConsumerPosition consumerPosition) { OffsetsInfo offsetsInfo; if (consumerPosition.getSeekTo() == null) { offsetsInfo = new OffsetsInfo(consumer, consumerPosition.getTopic()); } else { offsetsInfo = new OffsetsInfo(consumer, consumerPosition.getSeekTo().keySet()); } return new SeekOperations( consumer, offsetsInfo, getOffsetsForSeek(consumer, offsetsInfo, consumerPosition.getSeekType(), consumerPosition.getSeekTo()) ); } public void assignAndSeekNonEmptyPartitions() { consumer.assign(offsetsForSeek.keySet()); offsetsForSeek.forEach(consumer::seek); } public Map getBeginOffsets() { return offsetsInfo.getBeginOffsets(); } public Map getEndOffsets() { return offsetsInfo.getEndOffsets(); } public boolean assignedPartitionsFullyPolled() { return offsetsInfo.assignedPartitionsFullyPolled(); } // sum of (end - start) offsets for all partitions public long summaryOffsetsRange() { return offsetsInfo.summaryOffsetsRange(); } // sum of differences between initial consumer seek and current consumer position (across all partitions) public long offsetsProcessedFromSeek() { MutableLong count = new MutableLong(); offsetsForSeek.forEach((tp, initialOffset) -> count.add(consumer.position(tp) - initialOffset)); return count.getValue(); } // Get offsets to seek to. NOTE: offsets do not contain empty partitions offsets public Map getOffsetsForSeek() { return offsetsForSeek; } /** * Finds offsets for ConsumerPosition. Note: will return empty map if no offsets found for desired criteria. */ @VisibleForTesting static Map getOffsetsForSeek(Consumer consumer, OffsetsInfo offsetsInfo, SeekTypeDTO seekType, @Nullable Map seekTo) { switch (seekType) { case LATEST: return consumer.endOffsets(offsetsInfo.getNonEmptyPartitions()); case BEGINNING: return consumer.beginningOffsets(offsetsInfo.getNonEmptyPartitions()); case OFFSET: Preconditions.checkNotNull(seekTo); return fixOffsets(offsetsInfo, seekTo); case TIMESTAMP: Preconditions.checkNotNull(seekTo); return offsetsForTimestamp(consumer, offsetsInfo, seekTo); default: throw new IllegalStateException(); } } private static Map fixOffsets(OffsetsInfo offsetsInfo, Map offsets) { offsets = new HashMap<>(offsets); offsets.keySet().retainAll(offsetsInfo.getNonEmptyPartitions()); Map result = new HashMap<>(); offsets.forEach((tp, targetOffset) -> { long endOffset = offsetsInfo.getEndOffsets().get(tp); long beginningOffset = offsetsInfo.getBeginOffsets().get(tp); // fixing offsets with min - max bounds if (targetOffset > endOffset) { targetOffset = endOffset; } else if (targetOffset < beginningOffset) { targetOffset = beginningOffset; } result.put(tp, targetOffset); }); return result; } private static Map offsetsForTimestamp(Consumer consumer, OffsetsInfo offsetsInfo, Map timestamps) { timestamps = new HashMap<>(timestamps); timestamps.keySet().retainAll(offsetsInfo.getNonEmptyPartitions()); return consumer.offsetsForTimes(timestamps).entrySet().stream() .filter(e -> e.getValue() != null) .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/emitter/TailingEmitter.java ================================================ package com.provectus.kafka.ui.emitter; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import java.util.HashMap; import java.util.function.Predicate; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.errors.InterruptException; import reactor.core.publisher.FluxSink; @Slf4j public class TailingEmitter extends AbstractEmitter { private final Supplier consumerSupplier; private final ConsumerPosition consumerPosition; public TailingEmitter(Supplier consumerSupplier, ConsumerPosition consumerPosition, ConsumerRecordDeserializer deserializer, Predicate filter, PollingSettings pollingSettings) { super(new MessagesProcessing(deserializer, filter, false, null), pollingSettings); this.consumerSupplier = consumerSupplier; this.consumerPosition = consumerPosition; } @Override public void accept(FluxSink sink) { log.debug("Starting tailing polling for {}", consumerPosition); try (EnhancedConsumer consumer = consumerSupplier.get()) { assignAndSeek(consumer); while (!sink.isCancelled()) { sendPhase(sink, "Polling"); var polled = poll(sink, consumer); send(sink, polled); } sink.complete(); log.debug("Tailing finished"); } catch (InterruptException kafkaInterruptException) { log.debug("Tailing finished due to thread interruption"); sink.complete(); } catch (Exception e) { log.error("Error consuming {}", consumerPosition, e); sink.error(e); } } private void assignAndSeek(EnhancedConsumer consumer) { var seekOperations = SeekOperations.create(consumer, consumerPosition); var seekOffsets = new HashMap<>(seekOperations.getEndOffsets()); // defaulting offsets to topic end seekOffsets.putAll(seekOperations.getOffsetsForSeek()); // this will only set non-empty partitions consumer.assign(seekOffsets.keySet()); seekOffsets.forEach(consumer::seek); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ClusterNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class ClusterNotFoundException extends CustomBaseException { public ClusterNotFoundException() { super("Cluster not found"); } public ClusterNotFoundException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.CLUSTER_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ConnectNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class ConnectNotFoundException extends CustomBaseException { public ConnectNotFoundException() { super("Connect not found"); } @Override public ErrorCode getErrorCode() { return ErrorCode.CONNECT_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/CustomBaseException.java ================================================ package com.provectus.kafka.ui.exception; public abstract class CustomBaseException extends RuntimeException { protected CustomBaseException() { super(); } protected CustomBaseException(String message) { super(message); } protected CustomBaseException(String message, Throwable cause) { super(message, cause); } protected CustomBaseException(Throwable cause) { super(cause); } protected CustomBaseException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } public abstract ErrorCode getErrorCode(); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/DuplicateEntityException.java ================================================ package com.provectus.kafka.ui.exception; public class DuplicateEntityException extends CustomBaseException { public DuplicateEntityException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.DUPLICATED_ENTITY; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ErrorCode.java ================================================ package com.provectus.kafka.ui.exception; import java.util.HashSet; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; public enum ErrorCode { FORBIDDEN(403, HttpStatus.FORBIDDEN), UNEXPECTED(5000, HttpStatus.INTERNAL_SERVER_ERROR), KSQL_API_ERROR(5001, HttpStatus.INTERNAL_SERVER_ERROR), BINDING_FAIL(4001, HttpStatus.BAD_REQUEST), NOT_FOUND(404, HttpStatus.NOT_FOUND), VALIDATION_FAIL(4002, HttpStatus.BAD_REQUEST), READ_ONLY_MODE_ENABLE(4003, HttpStatus.METHOD_NOT_ALLOWED), CONNECT_CONFLICT_RESPONSE(4004, HttpStatus.CONFLICT), DUPLICATED_ENTITY(4005, HttpStatus.CONFLICT), UNPROCESSABLE_ENTITY(4006, HttpStatus.UNPROCESSABLE_ENTITY), CLUSTER_NOT_FOUND(4007, HttpStatus.NOT_FOUND), TOPIC_NOT_FOUND(4008, HttpStatus.NOT_FOUND), SCHEMA_NOT_FOUND(4009, HttpStatus.NOT_FOUND), CONNECT_NOT_FOUND(4010, HttpStatus.NOT_FOUND), KSQLDB_NOT_FOUND(4011, HttpStatus.NOT_FOUND), DIR_NOT_FOUND(4012, HttpStatus.BAD_REQUEST), TOPIC_OR_PARTITION_NOT_FOUND(4013, HttpStatus.BAD_REQUEST), INVALID_REQUEST(4014, HttpStatus.BAD_REQUEST), RECREATE_TOPIC_TIMEOUT(4015, HttpStatus.REQUEST_TIMEOUT), INVALID_ENTITY_STATE(4016, HttpStatus.BAD_REQUEST), SCHEMA_NOT_DELETED(4017, HttpStatus.INTERNAL_SERVER_ERROR), TOPIC_ANALYSIS_ERROR(4018, HttpStatus.BAD_REQUEST), FILE_UPLOAD_EXCEPTION(4019, HttpStatus.INTERNAL_SERVER_ERROR), ; static { // codes uniqueness check var codes = new HashSet(); for (ErrorCode value : ErrorCode.values()) { if (!codes.add(value.code())) { LoggerFactory.getLogger(ErrorCode.class) .warn("Multiple {} values refer to code {}", ErrorCode.class, value.code); } } } private final int code; private final HttpStatus httpStatus; ErrorCode(int code, HttpStatus httpStatus) { this.code = code; this.httpStatus = httpStatus; } public int code() { return code; } public HttpStatus httpStatus() { return httpStatus; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/FileUploadException.java ================================================ package com.provectus.kafka.ui.exception; import java.nio.file.Path; public class FileUploadException extends CustomBaseException { public FileUploadException(String msg, Throwable cause) { super(msg, cause); } public FileUploadException(Path path, Throwable cause) { super("Error uploading file %s".formatted(path), cause); } @Override public ErrorCode getErrorCode() { return ErrorCode.FILE_UPLOAD_EXCEPTION; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/GlobalErrorWebExceptionHandler.java ================================================ package com.provectus.kafka.ui.exception; import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.provectus.kafka.ui.model.ErrorResponseDTO; import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.boot.autoconfigure.web.WebProperties; import org.springframework.boot.autoconfigure.web.reactive.error.AbstractErrorWebExceptionHandler; import org.springframework.boot.web.reactive.error.ErrorAttributes; import org.springframework.context.ApplicationContext; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.codec.ServerCodecConfigurer; import org.springframework.stereotype.Component; import org.springframework.validation.FieldError; import org.springframework.web.bind.support.WebExchangeBindException; import org.springframework.web.reactive.function.server.RequestPredicates; import org.springframework.web.reactive.function.server.RouterFunction; import org.springframework.web.reactive.function.server.RouterFunctions; import org.springframework.web.reactive.function.server.ServerRequest; import org.springframework.web.reactive.function.server.ServerResponse; import org.springframework.web.server.ResponseStatusException; import reactor.core.publisher.Mono; @Component @Order(Ordered.HIGHEST_PRECEDENCE) public class GlobalErrorWebExceptionHandler extends AbstractErrorWebExceptionHandler { public GlobalErrorWebExceptionHandler(ErrorAttributes errorAttributes, ApplicationContext applicationContext, ServerCodecConfigurer codecConfigurer) { super(errorAttributes, new WebProperties.Resources(), applicationContext); this.setMessageWriters(codecConfigurer.getWriters()); } @Override protected RouterFunction getRoutingFunction(ErrorAttributes errorAttributes) { return RouterFunctions.route(RequestPredicates.all(), this::renderErrorResponse); } private Mono renderErrorResponse(ServerRequest request) { Throwable throwable = getError(request); // validation and params binding errors if (throwable instanceof WebExchangeBindException) { return render((WebExchangeBindException) throwable, request); } // requests mapping & access errors if (throwable instanceof ResponseStatusException) { return render((ResponseStatusException) throwable, request); } // custom exceptions if (throwable instanceof CustomBaseException) { return render((CustomBaseException) throwable, request); } return renderDefault(throwable, request); } private Mono renderDefault(Throwable throwable, ServerRequest request) { var response = new ErrorResponseDTO() .code(ErrorCode.UNEXPECTED.code()) .message(coalesce(throwable.getMessage(), "Unexpected internal error")) .requestId(requestId(request)) .timestamp(currentTimestamp()) .stackTrace(Throwables.getStackTraceAsString(throwable)); return ServerResponse .status(ErrorCode.UNEXPECTED.httpStatus()) .contentType(MediaType.APPLICATION_JSON) .bodyValue(response); } private Mono render(CustomBaseException baseException, ServerRequest request) { ErrorCode errorCode = baseException.getErrorCode(); var response = new ErrorResponseDTO() .code(errorCode.code()) .message(coalesce(baseException.getMessage(), "Internal error")) .requestId(requestId(request)) .timestamp(currentTimestamp()) .stackTrace(Throwables.getStackTraceAsString(baseException)); return ServerResponse .status(errorCode.httpStatus()) .contentType(MediaType.APPLICATION_JSON) .bodyValue(response); } private Mono render(WebExchangeBindException exception, ServerRequest request) { Map> fieldErrorsMap = exception.getFieldErrors().stream() .collect(Collectors .toMap(FieldError::getField, f -> Set.of(extractFieldErrorMsg(f)), Sets::union)); var fieldsErrors = fieldErrorsMap.entrySet().stream() .map(e -> { var err = new com.provectus.kafka.ui.model.FieldErrorDTO(); err.setFieldName(e.getKey()); err.setRestrictions(List.copyOf(e.getValue())); return err; }).toList(); var message = fieldsErrors.isEmpty() ? exception.getMessage() : "Fields validation failure"; var response = new ErrorResponseDTO() .code(ErrorCode.BINDING_FAIL.code()) .message(message) .requestId(requestId(request)) .timestamp(currentTimestamp()) .fieldsErrors(fieldsErrors) .stackTrace(Throwables.getStackTraceAsString(exception)); return ServerResponse .status(HttpStatus.BAD_REQUEST) .contentType(MediaType.APPLICATION_JSON) .bodyValue(response); } private Mono render(ResponseStatusException exception, ServerRequest request) { String msg = coalesce(exception.getReason(), exception.getMessage(), "Server error"); var response = new ErrorResponseDTO() .code(ErrorCode.UNEXPECTED.code()) .message(msg) .requestId(requestId(request)) .timestamp(currentTimestamp()) .stackTrace(Throwables.getStackTraceAsString(exception)); return ServerResponse .status(exception.getStatusCode()) .contentType(MediaType.APPLICATION_JSON) .bodyValue(response); } private String requestId(ServerRequest request) { return request.exchange().getRequest().getId(); } private BigDecimal currentTimestamp() { return BigDecimal.valueOf(System.currentTimeMillis()); } private String extractFieldErrorMsg(FieldError fieldError) { return coalesce(fieldError.getDefaultMessage(), fieldError.getCode(), "Invalid field value"); } private T coalesce(T... items) { return Stream.of(items).filter(Objects::nonNull).findFirst().orElse(null); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/IllegalEntityStateException.java ================================================ package com.provectus.kafka.ui.exception; public class IllegalEntityStateException extends CustomBaseException { public IllegalEntityStateException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.INVALID_ENTITY_STATE; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/InvalidRequestApiException.java ================================================ package com.provectus.kafka.ui.exception; public class InvalidRequestApiException extends CustomBaseException { public InvalidRequestApiException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.INVALID_REQUEST; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/JsonAvroConversionException.java ================================================ package com.provectus.kafka.ui.exception; public class JsonAvroConversionException extends ValidationException { public JsonAvroConversionException(String message) { super(message); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/KafkaConnectConflictReponseException.java ================================================ package com.provectus.kafka.ui.exception; import org.springframework.web.reactive.function.client.WebClientResponseException; public class KafkaConnectConflictReponseException extends CustomBaseException { public KafkaConnectConflictReponseException(WebClientResponseException.Conflict e) { super("Kafka Connect responded with 409 (Conflict) code. Response body: " + e.getResponseBodyAsString()); } @Override public ErrorCode getErrorCode() { return ErrorCode.CONNECT_CONFLICT_RESPONSE; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/KsqlApiException.java ================================================ package com.provectus.kafka.ui.exception; public class KsqlApiException extends CustomBaseException { public KsqlApiException(String message) { super(message); } public KsqlApiException(String message, Throwable cause) { super(message, cause); } @Override public ErrorCode getErrorCode() { return ErrorCode.KSQL_API_ERROR; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/KsqlDbNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class KsqlDbNotFoundException extends CustomBaseException { public KsqlDbNotFoundException() { super("KSQL DB not found"); } @Override public ErrorCode getErrorCode() { return ErrorCode.KSQLDB_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/LogDirNotFoundApiException.java ================================================ package com.provectus.kafka.ui.exception; public class LogDirNotFoundApiException extends CustomBaseException { public LogDirNotFoundApiException() { super("The user-specified log directory is not found in the broker config."); } @Override public ErrorCode getErrorCode() { return ErrorCode.DIR_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/NotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class NotFoundException extends CustomBaseException { public NotFoundException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ReadOnlyModeException.java ================================================ package com.provectus.kafka.ui.exception; public class ReadOnlyModeException extends CustomBaseException { public ReadOnlyModeException() { super("This cluster is in read-only mode."); } @Override public ErrorCode getErrorCode() { return ErrorCode.READ_ONLY_MODE_ENABLE; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/SchemaCompatibilityException.java ================================================ package com.provectus.kafka.ui.exception; public class SchemaCompatibilityException extends CustomBaseException { public SchemaCompatibilityException() { super("Schema being registered is incompatible with an earlier schema"); } @Override public ErrorCode getErrorCode() { return ErrorCode.UNPROCESSABLE_ENTITY; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/SchemaFailedToDeleteException.java ================================================ package com.provectus.kafka.ui.exception; public class SchemaFailedToDeleteException extends CustomBaseException { public SchemaFailedToDeleteException(String schemaName) { super(String.format("Unable to delete schema with name %s", schemaName)); } @Override public ErrorCode getErrorCode() { return ErrorCode.SCHEMA_NOT_DELETED; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/SchemaNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class SchemaNotFoundException extends CustomBaseException { public SchemaNotFoundException() { super("Schema not found"); } public SchemaNotFoundException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.SCHEMA_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/TopicAnalysisException.java ================================================ package com.provectus.kafka.ui.exception; public class TopicAnalysisException extends CustomBaseException { public TopicAnalysisException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.TOPIC_ANALYSIS_ERROR; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/TopicMetadataException.java ================================================ package com.provectus.kafka.ui.exception; public class TopicMetadataException extends CustomBaseException { public TopicMetadataException(String message) { super(message); } public TopicMetadataException(String message, Throwable cause) { super(message, cause); } @Override public ErrorCode getErrorCode() { return ErrorCode.INVALID_ENTITY_STATE; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/TopicNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class TopicNotFoundException extends CustomBaseException { public TopicNotFoundException() { super("Topic not found"); } @Override public ErrorCode getErrorCode() { return ErrorCode.TOPIC_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/TopicOrPartitionNotFoundException.java ================================================ package com.provectus.kafka.ui.exception; public class TopicOrPartitionNotFoundException extends CustomBaseException { public TopicOrPartitionNotFoundException() { super("This server does not host this topic-partition."); } @Override public ErrorCode getErrorCode() { return ErrorCode.TOPIC_OR_PARTITION_NOT_FOUND; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/TopicRecreationException.java ================================================ package com.provectus.kafka.ui.exception; public class TopicRecreationException extends CustomBaseException { @Override public ErrorCode getErrorCode() { return ErrorCode.RECREATE_TOPIC_TIMEOUT; } public TopicRecreationException(String topicName, int seconds) { super(String.format("Can't create topic '%s' in %d seconds: " + "topic deletion is still in progress", topicName, seconds)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/UnprocessableEntityException.java ================================================ package com.provectus.kafka.ui.exception; public class UnprocessableEntityException extends CustomBaseException { public UnprocessableEntityException(String message) { super(message); } @Override public ErrorCode getErrorCode() { return ErrorCode.UNPROCESSABLE_ENTITY; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ValidationException.java ================================================ package com.provectus.kafka.ui.exception; public class ValidationException extends CustomBaseException { public ValidationException(String message) { super(message); } public ValidationException(String message, Throwable cause) { super(message, cause); } @Override public ErrorCode getErrorCode() { return ErrorCode.VALIDATION_FAIL; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ClusterMapper.java ================================================ package com.provectus.kafka.ui.mapper; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.BrokerConfigDTO; import com.provectus.kafka.ui.model.BrokerDTO; import com.provectus.kafka.ui.model.BrokerDiskUsageDTO; import com.provectus.kafka.ui.model.BrokerMetricsDTO; import com.provectus.kafka.ui.model.ClusterDTO; import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.ClusterMetricsDTO; import com.provectus.kafka.ui.model.ClusterStatsDTO; import com.provectus.kafka.ui.model.ConfigSourceDTO; import com.provectus.kafka.ui.model.ConfigSynonymDTO; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.InternalBroker; import com.provectus.kafka.ui.model.InternalBrokerConfig; import com.provectus.kafka.ui.model.InternalBrokerDiskUsage; import com.provectus.kafka.ui.model.InternalClusterState; import com.provectus.kafka.ui.model.InternalPartition; import com.provectus.kafka.ui.model.InternalReplica; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.InternalTopicConfig; import com.provectus.kafka.ui.model.KafkaAclDTO; import com.provectus.kafka.ui.model.KafkaAclNamePatternTypeDTO; import com.provectus.kafka.ui.model.KafkaAclResourceTypeDTO; import com.provectus.kafka.ui.model.MetricDTO; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.PartitionDTO; import com.provectus.kafka.ui.model.ReplicaDTO; import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; import com.provectus.kafka.ui.model.TopicProducerStateDTO; import com.provectus.kafka.ui.service.metrics.RawMetric; import java.util.List; import java.util.Map; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.ProducerState; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @Mapper(componentModel = "spring") public interface ClusterMapper { ClusterDTO toCluster(InternalClusterState clusterState); ClusterStatsDTO toClusterStats(InternalClusterState clusterState); default ClusterMetricsDTO toClusterMetrics(Metrics metrics) { return new ClusterMetricsDTO() .items(metrics.getSummarizedMetrics().map(this::convert).toList()); } private MetricDTO convert(RawMetric rawMetric) { return new MetricDTO() .name(rawMetric.name()) .labels(rawMetric.labels()) .value(rawMetric.value()); } default BrokerMetricsDTO toBrokerMetrics(List metrics) { return new BrokerMetricsDTO() .metrics(metrics.stream().map(this::convert).toList()); } @Mapping(target = "isSensitive", source = "sensitive") @Mapping(target = "isReadOnly", source = "readOnly") BrokerConfigDTO toBrokerConfig(InternalBrokerConfig config); default ConfigSynonymDTO toConfigSynonym(ConfigEntry.ConfigSynonym config) { if (config == null) { return null; } ConfigSynonymDTO configSynonym = new ConfigSynonymDTO(); configSynonym.setName(config.name()); configSynonym.setValue(config.value()); if (config.source() != null) { configSynonym.setSource(ConfigSourceDTO.valueOf(config.source().name())); } return configSynonym; } TopicDTO toTopic(InternalTopic topic); PartitionDTO toPartition(InternalPartition topic); BrokerDTO toBrokerDto(InternalBroker broker); TopicDetailsDTO toTopicDetails(InternalTopic topic); @Mapping(target = "isReadOnly", source = "readOnly") @Mapping(target = "isSensitive", source = "sensitive") TopicConfigDTO toTopicConfig(InternalTopicConfig topic); ReplicaDTO toReplica(InternalReplica replica); ConnectDTO toKafkaConnect(ClustersProperties.ConnectCluster connect); List toFeaturesEnum(List features); default List map(Map map) { return map.values().stream().map(this::toPartition).toList(); } default BrokerDiskUsageDTO map(Integer id, InternalBrokerDiskUsage internalBrokerDiskUsage) { final BrokerDiskUsageDTO brokerDiskUsage = new BrokerDiskUsageDTO(); brokerDiskUsage.setBrokerId(id); brokerDiskUsage.segmentCount((int) internalBrokerDiskUsage.getSegmentCount()); brokerDiskUsage.segmentSize(internalBrokerDiskUsage.getSegmentSize()); return brokerDiskUsage; } default TopicProducerStateDTO map(int partition, ProducerState state) { return new TopicProducerStateDTO() .partition(partition) .producerId(state.producerId()) .producerEpoch(state.producerEpoch()) .lastSequence(state.lastSequence()) .lastTimestampMs(state.lastTimestamp()) .coordinatorEpoch(state.coordinatorEpoch().stream().boxed().findAny().orElse(null)) .currentTransactionStartOffset(state.currentTransactionStartOffset().stream().boxed().findAny().orElse(null)); } static KafkaAclDTO.OperationEnum mapAclOperation(AclOperation operation) { return switch (operation) { case ALL -> KafkaAclDTO.OperationEnum.ALL; case READ -> KafkaAclDTO.OperationEnum.READ; case WRITE -> KafkaAclDTO.OperationEnum.WRITE; case CREATE -> KafkaAclDTO.OperationEnum.CREATE; case DELETE -> KafkaAclDTO.OperationEnum.DELETE; case ALTER -> KafkaAclDTO.OperationEnum.ALTER; case DESCRIBE -> KafkaAclDTO.OperationEnum.DESCRIBE; case CLUSTER_ACTION -> KafkaAclDTO.OperationEnum.CLUSTER_ACTION; case DESCRIBE_CONFIGS -> KafkaAclDTO.OperationEnum.DESCRIBE_CONFIGS; case ALTER_CONFIGS -> KafkaAclDTO.OperationEnum.ALTER_CONFIGS; case IDEMPOTENT_WRITE -> KafkaAclDTO.OperationEnum.IDEMPOTENT_WRITE; case CREATE_TOKENS -> KafkaAclDTO.OperationEnum.CREATE_TOKENS; case DESCRIBE_TOKENS -> KafkaAclDTO.OperationEnum.DESCRIBE_TOKENS; case ANY -> throw new IllegalArgumentException("ANY operation can be only part of filter"); case UNKNOWN -> KafkaAclDTO.OperationEnum.UNKNOWN; }; } static KafkaAclResourceTypeDTO mapAclResourceType(ResourceType resourceType) { return switch (resourceType) { case CLUSTER -> KafkaAclResourceTypeDTO.CLUSTER; case TOPIC -> KafkaAclResourceTypeDTO.TOPIC; case GROUP -> KafkaAclResourceTypeDTO.GROUP; case DELEGATION_TOKEN -> KafkaAclResourceTypeDTO.DELEGATION_TOKEN; case TRANSACTIONAL_ID -> KafkaAclResourceTypeDTO.TRANSACTIONAL_ID; case USER -> KafkaAclResourceTypeDTO.USER; case ANY -> throw new IllegalArgumentException("ANY type can be only part of filter"); case UNKNOWN -> KafkaAclResourceTypeDTO.UNKNOWN; }; } static ResourceType mapAclResourceTypeDto(KafkaAclResourceTypeDTO dto) { return ResourceType.valueOf(dto.name()); } static PatternType mapPatternTypeDto(KafkaAclNamePatternTypeDTO dto) { return PatternType.valueOf(dto.name()); } static AclBinding toAclBinding(KafkaAclDTO dto) { return new AclBinding( new ResourcePattern( mapAclResourceTypeDto(dto.getResourceType()), dto.getResourceName(), mapPatternTypeDto(dto.getNamePatternType()) ), new AccessControlEntry( dto.getPrincipal(), dto.getHost(), AclOperation.valueOf(dto.getOperation().name()), AclPermissionType.valueOf(dto.getPermission().name()) ) ); } static KafkaAclDTO toKafkaAclDto(AclBinding binding) { var pattern = binding.pattern(); var filter = binding.toFilter().entryFilter(); return new KafkaAclDTO() .resourceType(mapAclResourceType(pattern.resourceType())) .resourceName(pattern.name()) .namePatternType(KafkaAclNamePatternTypeDTO.fromValue(pattern.patternType().name())) .principal(filter.principal()) .host(filter.host()) .operation(mapAclOperation(filter.operation())) .permission(KafkaAclDTO.PermissionEnum.fromValue(filter.permissionType().name())); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/ConsumerGroupMapper.java ================================================ package com.provectus.kafka.ui.mapper; import com.provectus.kafka.ui.model.BrokerDTO; import com.provectus.kafka.ui.model.ConsumerGroupDTO; import com.provectus.kafka.ui.model.ConsumerGroupDetailsDTO; import com.provectus.kafka.ui.model.ConsumerGroupStateDTO; import com.provectus.kafka.ui.model.ConsumerGroupTopicPartitionDTO; import com.provectus.kafka.ui.model.InternalConsumerGroup; import com.provectus.kafka.ui.model.InternalTopicConsumerGroup; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; public class ConsumerGroupMapper { private ConsumerGroupMapper() { } public static ConsumerGroupDTO toDto(InternalConsumerGroup c) { return convertToConsumerGroup(c, new ConsumerGroupDTO()); } public static ConsumerGroupDTO toDto(InternalTopicConsumerGroup c) { ConsumerGroupDTO consumerGroup = new ConsumerGroupDetailsDTO(); consumerGroup.setTopics(1); //for ui backward-compatibility, need to rm usage from ui consumerGroup.setGroupId(c.getGroupId()); consumerGroup.setMembers(c.getMembers()); consumerGroup.setConsumerLag(c.getConsumerLag()); consumerGroup.setSimple(c.isSimple()); consumerGroup.setPartitionAssignor(c.getPartitionAssignor()); consumerGroup.setState(mapConsumerGroupState(c.getState())); Optional.ofNullable(c.getCoordinator()) .ifPresent(cd -> consumerGroup.setCoordinator(mapCoordinator(cd))); return consumerGroup; } public static ConsumerGroupDetailsDTO toDetailsDto(InternalConsumerGroup g) { ConsumerGroupDetailsDTO details = convertToConsumerGroup(g, new ConsumerGroupDetailsDTO()); Map partitionMap = new HashMap<>(); for (Map.Entry entry : g.getOffsets().entrySet()) { ConsumerGroupTopicPartitionDTO partition = new ConsumerGroupTopicPartitionDTO(); partition.setTopic(entry.getKey().topic()); partition.setPartition(entry.getKey().partition()); partition.setCurrentOffset(entry.getValue()); final Optional endOffset = Optional.ofNullable(g.getEndOffsets()) .map(o -> o.get(entry.getKey())); final Long behind = endOffset.map(o -> o - entry.getValue()) .orElse(0L); partition.setEndOffset(endOffset.orElse(0L)); partition.setConsumerLag(behind); partitionMap.put(entry.getKey(), partition); } for (InternalConsumerGroup.InternalMember member : g.getMembers()) { for (TopicPartition topicPartition : member.getAssignment()) { final ConsumerGroupTopicPartitionDTO partition = partitionMap.computeIfAbsent( topicPartition, tp -> new ConsumerGroupTopicPartitionDTO() .topic(tp.topic()) .partition(tp.partition()) ); partition.setHost(member.getHost()); partition.setConsumerId(member.getConsumerId()); partitionMap.put(topicPartition, partition); } } details.setPartitions(new ArrayList<>(partitionMap.values())); return details; } private static T convertToConsumerGroup( InternalConsumerGroup c, T consumerGroup) { consumerGroup.setGroupId(c.getGroupId()); consumerGroup.setMembers(c.getMembers().size()); consumerGroup.setConsumerLag(c.getConsumerLag()); consumerGroup.setTopics(c.getTopicNum()); consumerGroup.setSimple(c.isSimple()); Optional.ofNullable(c.getState()) .ifPresent(s -> consumerGroup.setState(mapConsumerGroupState(s))); Optional.ofNullable(c.getCoordinator()) .ifPresent(cd -> consumerGroup.setCoordinator(mapCoordinator(cd))); consumerGroup.setPartitionAssignor(c.getPartitionAssignor()); return consumerGroup; } private static BrokerDTO mapCoordinator(Node node) { return new BrokerDTO().host(node.host()).id(node.id()).port(node.port()); } private static ConsumerGroupStateDTO mapConsumerGroupState( org.apache.kafka.common.ConsumerGroupState state) { switch (state) { case DEAD: return ConsumerGroupStateDTO.DEAD; case EMPTY: return ConsumerGroupStateDTO.EMPTY; case STABLE: return ConsumerGroupStateDTO.STABLE; case PREPARING_REBALANCE: return ConsumerGroupStateDTO.PREPARING_REBALANCE; case COMPLETING_REBALANCE: return ConsumerGroupStateDTO.COMPLETING_REBALANCE; default: return ConsumerGroupStateDTO.UNKNOWN; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/DescribeLogDirsMapper.java ================================================ package com.provectus.kafka.ui.mapper; import com.provectus.kafka.ui.model.BrokerTopicLogdirsDTO; import com.provectus.kafka.ui.model.BrokerTopicPartitionLogdirDTO; import com.provectus.kafka.ui.model.BrokersLogdirsDTO; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.springframework.stereotype.Component; @Component public class DescribeLogDirsMapper { public List toBrokerLogDirsList( Map> logDirsInfo) { return logDirsInfo.entrySet().stream().map( mapEntry -> mapEntry.getValue().entrySet().stream() .map(e -> toBrokerLogDirs(mapEntry.getKey(), e.getKey(), e.getValue())) .toList() ).flatMap(Collection::stream).collect(Collectors.toList()); } private BrokersLogdirsDTO toBrokerLogDirs(Integer broker, String dirName, DescribeLogDirsResponse.LogDirInfo logDirInfo) { BrokersLogdirsDTO result = new BrokersLogdirsDTO(); result.setName(dirName); if (logDirInfo.error != null && logDirInfo.error != Errors.NONE) { result.setError(logDirInfo.error.message()); } var topics = logDirInfo.replicaInfos.entrySet().stream() .collect(Collectors.groupingBy(e -> e.getKey().topic())).entrySet().stream() .map(e -> toTopicLogDirs(broker, e.getKey(), e.getValue())) .toList(); result.setTopics(topics); return result; } private BrokerTopicLogdirsDTO toTopicLogDirs(Integer broker, String name, List> partitions) { BrokerTopicLogdirsDTO topic = new BrokerTopicLogdirsDTO(); topic.setName(name); topic.setPartitions( partitions.stream().map( e -> topicPartitionLogDir( broker, e.getKey().partition(), e.getValue())).toList() ); return topic; } private BrokerTopicPartitionLogdirDTO topicPartitionLogDir(Integer broker, Integer partition, DescribeLogDirsResponse.ReplicaInfo replicaInfo) { BrokerTopicPartitionLogdirDTO logDir = new BrokerTopicPartitionLogdirDTO(); logDir.setBroker(broker); logDir.setPartition(partition); logDir.setSize(replicaInfo.size); logDir.setOffsetLag(replicaInfo.offsetLag); return logDir; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/KafkaConnectMapper.java ================================================ package com.provectus.kafka.ui.mapper; import com.provectus.kafka.ui.connect.model.ConnectorStatusConnector; import com.provectus.kafka.ui.connect.model.ConnectorTask; import com.provectus.kafka.ui.connect.model.NewConnector; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigValidationResponseDTO; import com.provectus.kafka.ui.model.ConnectorPluginDTO; import com.provectus.kafka.ui.model.ConnectorStatusDTO; import com.provectus.kafka.ui.model.ConnectorTaskStatusDTO; import com.provectus.kafka.ui.model.FullConnectorInfoDTO; import com.provectus.kafka.ui.model.TaskDTO; import com.provectus.kafka.ui.model.TaskStatusDTO; import com.provectus.kafka.ui.model.connect.InternalConnectInfo; import java.util.List; import org.mapstruct.Mapper; @Mapper(componentModel = "spring") public interface KafkaConnectMapper { NewConnector toClient(com.provectus.kafka.ui.model.NewConnectorDTO newConnector); ConnectorDTO fromClient(com.provectus.kafka.ui.connect.model.Connector connector); ConnectorStatusDTO fromClient(ConnectorStatusConnector connectorStatus); TaskDTO fromClient(ConnectorTask connectorTask); TaskStatusDTO fromClient(com.provectus.kafka.ui.connect.model.TaskStatus taskStatus); ConnectorPluginDTO fromClient( com.provectus.kafka.ui.connect.model.ConnectorPlugin connectorPlugin); ConnectorPluginConfigValidationResponseDTO fromClient( com.provectus.kafka.ui.connect.model.ConnectorPluginConfigValidationResponse connectorPluginConfigValidationResponse); default FullConnectorInfoDTO fullConnectorInfo(InternalConnectInfo connectInfo) { ConnectorDTO connector = connectInfo.getConnector(); List tasks = connectInfo.getTasks(); int failedTasksCount = (int) tasks.stream() .map(TaskDTO::getStatus) .map(TaskStatusDTO::getState) .filter(ConnectorTaskStatusDTO.FAILED::equals) .count(); return new FullConnectorInfoDTO() .connect(connector.getConnect()) .name(connector.getName()) .connectorClass((String) connectInfo.getConfig().get("connector.class")) .type(connector.getType()) .topics(connectInfo.getTopics()) .status(connector.getStatus()) .tasksCount(tasks.size()) .failedTasksCount(failedTasksCount); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/mapper/KafkaSrMapper.java ================================================ package com.provectus.kafka.ui.mapper; import com.provectus.kafka.ui.model.CompatibilityCheckResponseDTO; import com.provectus.kafka.ui.model.CompatibilityLevelDTO; import com.provectus.kafka.ui.model.NewSchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaReferenceDTO; import com.provectus.kafka.ui.model.SchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaTypeDTO; import com.provectus.kafka.ui.service.SchemaRegistryService; import com.provectus.kafka.ui.sr.model.Compatibility; import com.provectus.kafka.ui.sr.model.CompatibilityCheckResponse; import com.provectus.kafka.ui.sr.model.NewSubject; import com.provectus.kafka.ui.sr.model.SchemaReference; import com.provectus.kafka.ui.sr.model.SchemaType; import java.util.List; import java.util.Optional; import org.mapstruct.Mapper; @Mapper public interface KafkaSrMapper { default SchemaSubjectDTO toDto(SchemaRegistryService.SubjectWithCompatibilityLevel s) { return new SchemaSubjectDTO() .id(s.getId()) .version(s.getVersion()) .subject(s.getSubject()) .schema(s.getSchema()) .schemaType(SchemaTypeDTO.fromValue(Optional.ofNullable(s.getSchemaType()).orElse(SchemaType.AVRO).getValue())) .references(toDto(s.getReferences())) .compatibilityLevel(s.getCompatibility().toString()); } List toDto(List references); CompatibilityCheckResponseDTO toDto(CompatibilityCheckResponse ccr); CompatibilityLevelDTO.CompatibilityEnum toDto(Compatibility compatibility); NewSubject fromDto(NewSchemaSubjectDTO subjectDto); Compatibility fromDto(CompatibilityLevelDTO.CompatibilityEnum dtoEnum); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/BrokerMetrics.java ================================================ package com.provectus.kafka.ui.model; import java.util.List; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class BrokerMetrics { private final List metrics; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/CleanupPolicy.java ================================================ package com.provectus.kafka.ui.model; import java.util.Arrays; import java.util.Collections; import java.util.List; public enum CleanupPolicy { DELETE("delete"), COMPACT("compact"), COMPACT_DELETE(Arrays.asList("compact,delete", "delete,compact")), UNKNOWN("unknown"); private final List policies; CleanupPolicy(String policy) { this(Collections.singletonList(policy)); } CleanupPolicy(List policies) { this.policies = policies; } public String getPolicy() { return policies.get(0); } public static CleanupPolicy fromString(String string) { return Arrays.stream(CleanupPolicy.values()) .filter(v -> v.policies.stream().anyMatch( s -> s.equals(string.replace(" ", "") ) ) ).findFirst() .orElse(UNKNOWN); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ClusterFeature.java ================================================ package com.provectus.kafka.ui.model; public enum ClusterFeature { KAFKA_CONNECT, KSQL_DB, SCHEMA_REGISTRY, TOPIC_DELETION, KAFKA_ACL_VIEW, KAFKA_ACL_EDIT } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/ConsumerPosition.java ================================================ package com.provectus.kafka.ui.model; import java.util.Map; import javax.annotation.Nullable; import lombok.Value; import org.apache.kafka.common.TopicPartition; @Value public class ConsumerPosition { SeekTypeDTO seekType; String topic; @Nullable Map seekTo; // null if positioning should apply to all tps } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBroker.java ================================================ package com.provectus.kafka.ui.model; import java.math.BigDecimal; import javax.annotation.Nullable; import lombok.Data; import org.apache.kafka.common.Node; @Data public class InternalBroker { private final Integer id; private final String host; private final Integer port; private final @Nullable BigDecimal bytesInPerSec; private final @Nullable BigDecimal bytesOutPerSec; private final @Nullable Integer partitionsLeader; private final @Nullable Integer partitions; private final @Nullable Integer inSyncPartitions; private final @Nullable BigDecimal leadersSkew; private final @Nullable BigDecimal partitionsSkew; public InternalBroker(Node node, PartitionDistributionStats partitionDistribution, Statistics statistics) { this.id = node.id(); this.host = node.host(); this.port = node.port(); this.bytesInPerSec = statistics.getMetrics().getBrokerBytesInPerSec().get(node.id()); this.bytesOutPerSec = statistics.getMetrics().getBrokerBytesOutPerSec().get(node.id()); this.partitionsLeader = partitionDistribution.getPartitionLeaders().get(node); this.partitions = partitionDistribution.getPartitionsCount().get(node); this.inSyncPartitions = partitionDistribution.getInSyncPartitions().get(node); this.leadersSkew = partitionDistribution.leadersSkew(node); this.partitionsSkew = partitionDistribution.partitionsSkew(node); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBrokerConfig.java ================================================ package com.provectus.kafka.ui.model; import java.util.List; import lombok.Builder; import lombok.Data; import org.apache.kafka.clients.admin.ConfigEntry; @Data @Builder public class InternalBrokerConfig { private final String name; private final String value; private final ConfigEntry.ConfigSource source; private final boolean isSensitive; private final boolean isReadOnly; private final List synonyms; public static InternalBrokerConfig from(ConfigEntry configEntry) { InternalBrokerConfig.InternalBrokerConfigBuilder builder = InternalBrokerConfig.builder() .name(configEntry.name()) .value(configEntry.value()) .source(configEntry.source()) .isReadOnly(configEntry.isReadOnly()) .isSensitive(configEntry.isSensitive()) .synonyms(configEntry.synonyms()); return builder.build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalBrokerDiskUsage.java ================================================ package com.provectus.kafka.ui.model; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class InternalBrokerDiskUsage { private final long segmentCount; private final long segmentSize; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterMetrics.java ================================================ package com.provectus.kafka.ui.model; import java.math.BigDecimal; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class InternalClusterMetrics { public static InternalClusterMetrics empty() { return InternalClusterMetrics.builder() .brokers(List.of()) .topics(Map.of()) .status(ServerStatusDTO.OFFLINE) .internalBrokerMetrics(Map.of()) .metrics(List.of()) .version("unknown") .build(); } private final String version; private final ServerStatusDTO status; private final Throwable lastKafkaException; private final int brokerCount; private final int activeControllers; private final List brokers; private final int topicCount; private final Map topics; // partitions stats private final int underReplicatedPartitionCount; private final int onlinePartitionCount; private final int offlinePartitionCount; private final int inSyncReplicasCount; private final int outOfSyncReplicasCount; // log dir stats @Nullable // will be null if log dir collection disabled private final Map internalBrokerDiskUsage; // metrics from metrics collector private final BigDecimal bytesInPerSec; private final BigDecimal bytesOutPerSec; private final Map internalBrokerMetrics; private final List metrics; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalClusterState.java ================================================ package com.provectus.kafka.ui.model; import com.google.common.base.Throwables; import java.math.BigDecimal; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import lombok.Data; import org.apache.kafka.common.Node; @Data public class InternalClusterState { private String name; private ServerStatusDTO status; private MetricsCollectionErrorDTO lastError; private Integer topicCount; private Integer brokerCount; private Integer activeControllers; private Integer onlinePartitionCount; private Integer offlinePartitionCount; private Integer inSyncReplicasCount; private Integer outOfSyncReplicasCount; private Integer underReplicatedPartitionCount; private List diskUsage; private String version; private List features; private BigDecimal bytesInPerSec; private BigDecimal bytesOutPerSec; private Boolean readOnly; public InternalClusterState(KafkaCluster cluster, Statistics statistics) { name = cluster.getName(); status = statistics.getStatus(); lastError = Optional.ofNullable(statistics.getLastKafkaException()) .map(e -> new MetricsCollectionErrorDTO() .message(e.getMessage()) .stackTrace(Throwables.getStackTraceAsString(e))) .orElse(null); topicCount = statistics.getTopicDescriptions().size(); brokerCount = statistics.getClusterDescription().getNodes().size(); activeControllers = Optional.ofNullable(statistics.getClusterDescription().getController()) .map(Node::id) .orElse(null); version = statistics.getVersion(); if (statistics.getLogDirInfo() != null) { diskUsage = statistics.getLogDirInfo().getBrokerStats().entrySet().stream() .map(e -> new BrokerDiskUsageDTO() .brokerId(e.getKey()) .segmentSize(e.getValue().getSegmentSize()) .segmentCount(e.getValue().getSegmentsCount())) .collect(Collectors.toList()); } features = statistics.getFeatures(); bytesInPerSec = statistics .getMetrics() .getBrokerBytesInPerSec() .values().stream() .reduce(BigDecimal::add) .orElse(null); bytesOutPerSec = statistics .getMetrics() .getBrokerBytesOutPerSec() .values().stream() .reduce(BigDecimal::add) .orElse(null); var partitionsStats = new PartitionsStats(statistics.getTopicDescriptions().values()); onlinePartitionCount = partitionsStats.getOnlinePartitionCount(); offlinePartitionCount = partitionsStats.getOfflinePartitionCount(); inSyncReplicasCount = partitionsStats.getInSyncReplicasCount(); outOfSyncReplicasCount = partitionsStats.getOutOfSyncReplicasCount(); underReplicatedPartitionCount = partitionsStats.getUnderReplicatedPartitionCount(); readOnly = cluster.isReadOnly(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalConsumerGroup.java ================================================ package com.provectus.kafka.ui.model; import java.util.Collection; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.Builder; import lombok.Data; import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.common.ConsumerGroupState; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; @Data @Builder(toBuilder = true) public class InternalConsumerGroup { private final String groupId; private final boolean simple; private final Collection members; private final Map offsets; private final Map endOffsets; private final Long consumerLag; private final Integer topicNum; private final String partitionAssignor; private final ConsumerGroupState state; private final Node coordinator; @Data @Builder(toBuilder = true) public static class InternalMember { private final String consumerId; private final String groupInstanceId; private final String clientId; private final String host; private final Set assignment; } public static InternalConsumerGroup create( ConsumerGroupDescription description, Map groupOffsets, Map topicEndOffsets) { var builder = InternalConsumerGroup.builder(); builder.groupId(description.groupId()); builder.simple(description.isSimpleConsumerGroup()); builder.state(description.state()); builder.partitionAssignor(description.partitionAssignor()); Collection internalMembers = initInternalMembers(description); builder.members(internalMembers); builder.offsets(groupOffsets); builder.endOffsets(topicEndOffsets); builder.consumerLag(calculateConsumerLag(groupOffsets, topicEndOffsets)); builder.topicNum(calculateTopicNum(groupOffsets, internalMembers)); Optional.ofNullable(description.coordinator()).ifPresent(builder::coordinator); return builder.build(); } private static Long calculateConsumerLag(Map offsets, Map endOffsets) { Long consumerLag = null; // consumerLag should be undefined if no committed offsets found for topic if (!offsets.isEmpty()) { consumerLag = offsets.entrySet().stream() .mapToLong(e -> Optional.ofNullable(endOffsets) .map(o -> o.get(e.getKey())) .map(o -> o - e.getValue()) .orElse(0L) ).sum(); } return consumerLag; } private static Integer calculateTopicNum(Map offsets, Collection members) { return (int) Stream.concat( offsets.keySet().stream().map(TopicPartition::topic), members.stream() .flatMap(m -> m.getAssignment().stream().map(TopicPartition::topic)) ).distinct().count(); } private static Collection initInternalMembers(ConsumerGroupDescription description) { return description.members().stream() .map(m -> InternalConsumerGroup.InternalMember.builder() .assignment(m.assignment().topicPartitions()) .clientId(m.clientId()) .groupInstanceId(m.groupInstanceId().orElse("")) .consumerId(m.consumerId()) .clientId(m.clientId()) .host(m.host()) .build() ).collect(Collectors.toList()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalLogDirStats.java ================================================ package com.provectus.kafka.ui.model; import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summarizingLong; import static java.util.stream.Collectors.toList; import java.util.List; import java.util.LongSummaryStatistics; import java.util.Map; import lombok.Value; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.requests.DescribeLogDirsResponse; import reactor.util.function.Tuple2; import reactor.util.function.Tuple3; import reactor.util.function.Tuples; @Value public class InternalLogDirStats { @Value public static class SegmentStats { long segmentSize; int segmentsCount; public SegmentStats(LongSummaryStatistics s) { segmentSize = s.getSum(); segmentsCount = (int) s.getCount(); } } Map partitionsStats; Map topicStats; Map brokerStats; public static InternalLogDirStats empty() { return new InternalLogDirStats(Map.of()); } public InternalLogDirStats(Map> log) { final List> topicPartitions = log.entrySet().stream().flatMap(b -> b.getValue().entrySet().stream().flatMap(topicMap -> topicMap.getValue().replicaInfos.entrySet().stream() .map(e -> Tuples.of(b.getKey(), e.getKey(), e.getValue().size)) ) ).toList(); partitionsStats = topicPartitions.stream().collect( groupingBy( Tuple2::getT2, collectingAndThen( summarizingLong(Tuple3::getT3), SegmentStats::new))); topicStats = topicPartitions.stream().collect( groupingBy( t -> t.getT2().topic(), collectingAndThen( summarizingLong(Tuple3::getT3), SegmentStats::new))); brokerStats = topicPartitions.stream().collect( groupingBy( Tuple2::getT1, collectingAndThen( summarizingLong(Tuple3::getT3), SegmentStats::new))); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartition.java ================================================ package com.provectus.kafka.ui.model; import java.util.List; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class InternalPartition { private final int partition; private final Integer leader; private final List replicas; private final int inSyncReplicasCount; private final int replicasCount; private final Long offsetMin; private final Long offsetMax; // from log dir private final Long segmentSize; private final Integer segmentCount; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalPartitionsOffsets.java ================================================ package com.provectus.kafka.ui.model; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Table; import java.util.Map; import java.util.Optional; import lombok.Value; import org.apache.kafka.common.TopicPartition; public class InternalPartitionsOffsets { @Value public static class Offsets { Long earliest; Long latest; } private final Table offsets = HashBasedTable.create(); public InternalPartitionsOffsets(Map offsetsMap) { offsetsMap.forEach((tp, o) -> this.offsets.put(tp.topic(), tp.partition(), o)); } public static InternalPartitionsOffsets empty() { return new InternalPartitionsOffsets(Map.of()); } public Optional get(String topic, int partition) { return Optional.ofNullable(offsets.get(topic, partition)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalReplica.java ================================================ package com.provectus.kafka.ui.model; import lombok.Builder; import lombok.Data; import lombok.RequiredArgsConstructor; @Data @Builder @RequiredArgsConstructor public class InternalReplica { private final int broker; private final boolean leader; private final boolean inSync; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalSegmentSizeDto.java ================================================ package com.provectus.kafka.ui.model; import java.util.Map; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class InternalSegmentSizeDto { private final Map internalTopicWithSegmentSize; private final InternalClusterMetrics clusterMetricsWithSegmentSize; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopic.java ================================================ package com.provectus.kafka.ui.model; import com.provectus.kafka.ui.config.ClustersProperties; import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.Builder; import lombok.Data; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.TopicPartition; @Data @Builder(toBuilder = true) public class InternalTopic { ClustersProperties clustersProperties; // from TopicDescription private final String name; private final boolean internal; private final int replicas; private final int partitionCount; private final int inSyncReplicas; private final int replicationFactor; private final int underReplicatedPartitions; private final Map partitions; // topic configs private final List topicConfigs; private final CleanupPolicy cleanUpPolicy; // rates from metrics private final BigDecimal bytesInPerSec; private final BigDecimal bytesOutPerSec; // from log dir data private final long segmentSize; private final long segmentCount; public static InternalTopic from(TopicDescription topicDescription, List configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, InternalLogDirStats logDirInfo, @Nullable String internalTopicPrefix) { var topic = InternalTopic.builder(); internalTopicPrefix = internalTopicPrefix == null || internalTopicPrefix.isEmpty() ? "_" : internalTopicPrefix; topic.internal( topicDescription.isInternal() || topicDescription.name().startsWith(internalTopicPrefix) ); topic.name(topicDescription.name()); List partitions = topicDescription.partitions().stream() .map(partition -> { var partitionDto = InternalPartition.builder(); partitionDto.leader(partition.leader() != null ? partition.leader().id() : null); partitionDto.partition(partition.partition()); partitionDto.inSyncReplicasCount(partition.isr().size()); partitionDto.replicasCount(partition.replicas().size()); List replicas = partition.replicas().stream() .map(r -> InternalReplica.builder() .broker(r.id()) .inSync(partition.isr().contains(r)) .leader(partition.leader() != null && partition.leader().id() == r.id()) .build()) .collect(Collectors.toList()); partitionDto.replicas(replicas); partitionsOffsets.get(topicDescription.name(), partition.partition()) .ifPresent(offsets -> { partitionDto.offsetMin(offsets.getEarliest()); partitionDto.offsetMax(offsets.getLatest()); }); var segmentStats = logDirInfo.getPartitionsStats().get( new TopicPartition(topicDescription.name(), partition.partition())); if (segmentStats != null) { partitionDto.segmentCount(segmentStats.getSegmentsCount()); partitionDto.segmentSize(segmentStats.getSegmentSize()); } return partitionDto.build(); }) .toList(); topic.partitions(partitions.stream().collect( Collectors.toMap(InternalPartition::getPartition, t -> t))); var partitionsStats = new PartitionsStats(topicDescription); topic.replicas(partitionsStats.getReplicasCount()); topic.partitionCount(partitionsStats.getPartitionsCount()); topic.inSyncReplicas(partitionsStats.getInSyncReplicasCount()); topic.underReplicatedPartitions(partitionsStats.getUnderReplicatedPartitionCount()); topic.replicationFactor( topicDescription.partitions().isEmpty() ? 0 : topicDescription.partitions().get(0).replicas().size() ); var segmentStats = logDirInfo.getTopicStats().get(topicDescription.name()); if (segmentStats != null) { topic.segmentCount(segmentStats.getSegmentsCount()); topic.segmentSize(segmentStats.getSegmentSize()); } topic.bytesInPerSec(metrics.getTopicBytesInPerSec().get(topicDescription.name())); topic.bytesOutPerSec(metrics.getTopicBytesOutPerSec().get(topicDescription.name())); topic.topicConfigs( configs.stream().map(InternalTopicConfig::from).collect(Collectors.toList())); topic.cleanUpPolicy( configs.stream() .filter(config -> config.name().equals("cleanup.policy")) .findFirst() .map(ConfigEntry::value) .map(CleanupPolicy::fromString) .orElse(CleanupPolicy.UNKNOWN) ); return topic.build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopicConfig.java ================================================ package com.provectus.kafka.ui.model; import java.util.List; import lombok.Builder; import lombok.Data; import org.apache.kafka.clients.admin.ConfigEntry; @Data @Builder public class InternalTopicConfig { private final String name; private final String value; private final String defaultValue; private final ConfigEntry.ConfigSource source; private final boolean isSensitive; private final boolean isReadOnly; private final List synonyms; private final String doc; public static InternalTopicConfig from(ConfigEntry configEntry) { InternalTopicConfig.InternalTopicConfigBuilder builder = InternalTopicConfig.builder() .name(configEntry.name()) .value(configEntry.value()) .source(configEntry.source()) .isReadOnly(configEntry.isReadOnly()) .isSensitive(configEntry.isSensitive()) .synonyms(configEntry.synonyms()) .doc(configEntry.documentation()); if (configEntry.source() == ConfigEntry.ConfigSource.DEFAULT_CONFIG) { // this is important case, because for some configs like "confluent.*" no synonyms returned, but // they are set by default and "source" == DEFAULT_CONFIG builder.defaultValue(configEntry.value()); } else { // normally by default first entity of synonyms values will be used. configEntry.synonyms().stream() // skipping DYNAMIC_TOPIC_CONFIG value - which is explicitly set value when // topic was created (not default), see ConfigEntry.synonyms() doc .filter(s -> s.source() != ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG) .map(ConfigEntry.ConfigSynonym::value) .findFirst() .ifPresent(builder::defaultValue); } return builder.build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/InternalTopicConsumerGroup.java ================================================ package com.provectus.kafka.ui.model; import java.util.Map; import java.util.Optional; import javax.annotation.Nullable; import lombok.Builder; import lombok.Value; import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.common.ConsumerGroupState; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; @Value @Builder public class InternalTopicConsumerGroup { String groupId; int members; @Nullable Long consumerLag; //null means no committed offsets found for this group boolean isSimple; String partitionAssignor; ConsumerGroupState state; @Nullable Node coordinator; public static InternalTopicConsumerGroup create( String topic, ConsumerGroupDescription g, Map committedOffsets, Map endOffsets) { return InternalTopicConsumerGroup.builder() .groupId(g.groupId()) .members( (int) g.members().stream() // counting only members with target topic assignment .filter(m -> m.assignment().topicPartitions().stream().anyMatch(p -> p.topic().equals(topic))) .count() ) .consumerLag(calculateConsumerLag(committedOffsets, endOffsets)) .isSimple(g.isSimpleConsumerGroup()) .partitionAssignor(g.partitionAssignor()) .state(g.state()) .coordinator(g.coordinator()) .build(); } @Nullable private static Long calculateConsumerLag(Map committedOffsets, Map endOffsets) { if (committedOffsets.isEmpty()) { return null; } return committedOffsets.entrySet().stream() .mapToLong(e -> Optional.ofNullable(endOffsets.get(e.getKey())) .map(o -> o - e.getValue()) .orElse(0L) ).sum(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/KafkaCluster.java ================================================ package com.provectus.kafka.ui.model; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.emitter.PollingSettings; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import com.provectus.kafka.ui.service.masking.DataMasking; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.Map; import java.util.Properties; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) @AllArgsConstructor(access = AccessLevel.PRIVATE) public class KafkaCluster { private final ClustersProperties.Cluster originalProperties; private final String name; private final String version; private final String bootstrapServers; private final Properties properties; private final boolean readOnly; private final MetricsConfig metricsConfig; private final DataMasking masking; private final PollingSettings pollingSettings; private final ReactiveFailover schemaRegistryClient; private final Map> connectsClients; private final ReactiveFailover ksqlClient; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Metrics.java ================================================ package com.provectus.kafka.ui.model; import static java.util.stream.Collectors.toMap; import com.provectus.kafka.ui.service.metrics.RawMetric; import java.math.BigDecimal; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Stream; import lombok.Builder; import lombok.Value; @Builder @Value public class Metrics { Map brokerBytesInPerSec; Map brokerBytesOutPerSec; Map topicBytesInPerSec; Map topicBytesOutPerSec; Map> perBrokerMetrics; public static Metrics empty() { return Metrics.builder() .brokerBytesInPerSec(Map.of()) .brokerBytesOutPerSec(Map.of()) .topicBytesInPerSec(Map.of()) .topicBytesOutPerSec(Map.of()) .perBrokerMetrics(Map.of()) .build(); } public Stream getSummarizedMetrics() { return perBrokerMetrics.values().stream() .flatMap(Collection::stream) .collect(toMap(RawMetric::identityKey, m -> m, (m1, m2) -> m1.copyWithValue(m1.value().add(m2.value())))) .values() .stream(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/MetricsConfig.java ================================================ package com.provectus.kafka.ui.model; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) @AllArgsConstructor(access = AccessLevel.PRIVATE) public class MetricsConfig { public static final String JMX_METRICS_TYPE = "JMX"; public static final String PROMETHEUS_METRICS_TYPE = "PROMETHEUS"; private final String type; private final Integer port; private final boolean ssl; private final String username; private final String password; private final String keystoreLocation; private final String keystorePassword; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionDistributionStats.java ================================================ package com.provectus.kafka.ui.model; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionInfo; @RequiredArgsConstructor(access = AccessLevel.PRIVATE) @Getter @Slf4j public class PartitionDistributionStats { // avg skew will show unuseful results on low number of partitions private static final int MIN_PARTITIONS_FOR_SKEW_CALCULATION = 50; private final Map partitionLeaders; private final Map partitionsCount; private final Map inSyncPartitions; private final double avgLeadersCntPerBroker; private final double avgPartitionsPerBroker; private final boolean skewCanBeCalculated; public static PartitionDistributionStats create(Statistics stats) { return create(stats, MIN_PARTITIONS_FOR_SKEW_CALCULATION); } static PartitionDistributionStats create(Statistics stats, int minPartitionsForSkewCalculation) { var partitionLeaders = new HashMap(); var partitionsReplicated = new HashMap(); var isr = new HashMap(); int partitionsCnt = 0; for (TopicDescription td : stats.getTopicDescriptions().values()) { for (TopicPartitionInfo tp : td.partitions()) { partitionsCnt++; tp.replicas().forEach(r -> incr(partitionsReplicated, r)); tp.isr().forEach(r -> incr(isr, r)); if (tp.leader() != null) { incr(partitionLeaders, tp.leader()); } } } int nodesWithPartitions = partitionsReplicated.size(); int partitionReplications = partitionsReplicated.values().stream().mapToInt(i -> i).sum(); var avgPartitionsPerBroker = nodesWithPartitions == 0 ? 0 : ((double) partitionReplications) / nodesWithPartitions; int nodesWithLeaders = partitionLeaders.size(); int leadersCnt = partitionLeaders.values().stream().mapToInt(i -> i).sum(); var avgLeadersCntPerBroker = nodesWithLeaders == 0 ? 0 : ((double) leadersCnt) / nodesWithLeaders; return new PartitionDistributionStats( partitionLeaders, partitionsReplicated, isr, avgLeadersCntPerBroker, avgPartitionsPerBroker, partitionsCnt >= minPartitionsForSkewCalculation ); } private static void incr(Map map, Node n) { map.compute(n, (k, c) -> c == null ? 1 : ++c); } @Nullable public BigDecimal partitionsSkew(Node node) { return calculateAvgSkew(partitionsCount.get(node), avgPartitionsPerBroker); } @Nullable public BigDecimal leadersSkew(Node node) { return calculateAvgSkew(partitionLeaders.get(node), avgLeadersCntPerBroker); } // Returns difference (in percents) from average value, null if it can't be calculated @Nullable private BigDecimal calculateAvgSkew(@Nullable Integer value, double avgValue) { if (avgValue == 0 || !skewCanBeCalculated) { return null; } value = value == null ? 0 : value; return new BigDecimal((value - avgValue) / avgValue * 100.0) .setScale(1, RoundingMode.HALF_UP); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/PartitionsStats.java ================================================ package com.provectus.kafka.ui.model; import java.util.Collection; import java.util.List; import lombok.Data; import org.apache.kafka.clients.admin.TopicDescription; @Data public class PartitionsStats { private int partitionsCount; private int replicasCount; private int onlinePartitionCount; private int offlinePartitionCount; private int inSyncReplicasCount; private int outOfSyncReplicasCount; private int underReplicatedPartitionCount; public PartitionsStats(TopicDescription description) { this(List.of(description)); } public PartitionsStats(Collection topicDescriptions) { topicDescriptions.stream() .flatMap(t -> t.partitions().stream()) .forEach(p -> { partitionsCount++; replicasCount += p.replicas().size(); onlinePartitionCount += p.leader() != null ? 1 : 0; offlinePartitionCount += p.leader() == null ? 1 : 0; inSyncReplicasCount += p.isr().size(); outOfSyncReplicasCount += (p.replicas().size() - p.isr().size()); if (p.replicas().size() > p.isr().size()) { underReplicatedPartitionCount++; } }); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/Statistics.java ================================================ package com.provectus.kafka.ui.model; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.util.List; import java.util.Map; import java.util.Set; import lombok.Builder; import lombok.Value; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; @Value @Builder(toBuilder = true) public class Statistics { ServerStatusDTO status; Throwable lastKafkaException; String version; List features; ReactiveAdminClient.ClusterDescription clusterDescription; Metrics metrics; InternalLogDirStats logDirInfo; Map topicDescriptions; Map> topicConfigs; public static Statistics empty() { return builder() .status(ServerStatusDTO.OFFLINE) .version("Unknown") .features(List.of()) .clusterDescription( new ReactiveAdminClient.ClusterDescription(null, null, List.of(), Set.of())) .metrics(Metrics.empty()) .logDirInfo(InternalLogDirStats.empty()) .topicDescriptions(Map.of()) .topicConfigs(Map.of()) .build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/connect/InternalConnectInfo.java ================================================ package com.provectus.kafka.ui.model.connect; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.TaskDTO; import java.util.List; import java.util.Map; import lombok.Builder; import lombok.Data; @Data @Builder(toBuilder = true) public class InternalConnectInfo { private final ConnectorDTO connector; private final Map config; private final List tasks; private final List topics; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/AccessContext.java ================================================ package com.provectus.kafka.ui.model.rbac; import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction; import com.provectus.kafka.ui.model.rbac.permission.AuditAction; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.KsqlAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import lombok.Value; import org.springframework.util.Assert; @Value public class AccessContext { Collection applicationConfigActions; String cluster; Collection clusterConfigActions; String topic; Collection topicActions; String consumerGroup; Collection consumerGroupActions; String connect; Collection connectActions; String connector; String schema; Collection schemaActions; Collection ksqlActions; Collection aclActions; Collection auditAction; String operationName; Object operationParams; public static AccessContextBuilder builder() { return new AccessContextBuilder(); } public static final class AccessContextBuilder { private static final String ACTIONS_NOT_PRESENT = "actions not present"; private Collection applicationConfigActions = Collections.emptySet(); private String cluster; private Collection clusterConfigActions = Collections.emptySet(); private String topic; private Collection topicActions = Collections.emptySet(); private String consumerGroup; private Collection consumerGroupActions = Collections.emptySet(); private String connect; private Collection connectActions = Collections.emptySet(); private String connector; private String schema; private Collection schemaActions = Collections.emptySet(); private Collection ksqlActions = Collections.emptySet(); private Collection aclActions = Collections.emptySet(); private Collection auditActions = Collections.emptySet(); private String operationName; private Object operationParams; private AccessContextBuilder() { } public AccessContextBuilder applicationConfigActions(ApplicationConfigAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.applicationConfigActions = List.of(actions); return this; } public AccessContextBuilder cluster(String cluster) { this.cluster = cluster; return this; } public AccessContextBuilder clusterConfigActions(ClusterConfigAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.clusterConfigActions = List.of(actions); return this; } public AccessContextBuilder topic(String topic) { this.topic = topic; return this; } public AccessContextBuilder topicActions(TopicAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.topicActions = List.of(actions); return this; } public AccessContextBuilder consumerGroup(String consumerGroup) { this.consumerGroup = consumerGroup; return this; } public AccessContextBuilder consumerGroupActions(ConsumerGroupAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.consumerGroupActions = List.of(actions); return this; } public AccessContextBuilder connect(String connect) { this.connect = connect; return this; } public AccessContextBuilder connectActions(ConnectAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.connectActions = List.of(actions); return this; } public AccessContextBuilder connector(String connector) { this.connector = connector; return this; } public AccessContextBuilder schema(String schema) { this.schema = schema; return this; } public AccessContextBuilder schemaActions(SchemaAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.schemaActions = List.of(actions); return this; } public AccessContextBuilder ksqlActions(KsqlAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.ksqlActions = List.of(actions); return this; } public AccessContextBuilder aclActions(AclAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.aclActions = List.of(actions); return this; } public AccessContextBuilder auditActions(AuditAction... actions) { Assert.isTrue(actions.length > 0, ACTIONS_NOT_PRESENT); this.auditActions = List.of(actions); return this; } public AccessContextBuilder operationName(String operationName) { this.operationName = operationName; return this; } public AccessContextBuilder operationParams(Object operationParams) { this.operationParams = operationParams; return this; } public AccessContextBuilder operationParams(Map paramsMap) { this.operationParams = paramsMap; return this; } public AccessContext build() { return new AccessContext( applicationConfigActions, cluster, clusterConfigActions, topic, topicActions, consumerGroup, consumerGroupActions, connect, connectActions, connector, schema, schemaActions, ksqlActions, aclActions, auditActions, operationName, operationParams); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Permission.java ================================================ package com.provectus.kafka.ui.model.rbac; import static com.provectus.kafka.ui.model.rbac.Resource.ACL; import static com.provectus.kafka.ui.model.rbac.Resource.APPLICATIONCONFIG; import static com.provectus.kafka.ui.model.rbac.Resource.AUDIT; import static com.provectus.kafka.ui.model.rbac.Resource.CLUSTERCONFIG; import static com.provectus.kafka.ui.model.rbac.Resource.KSQL; import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.model.rbac.permission.ApplicationConfigAction; import com.provectus.kafka.ui.model.rbac.permission.AuditAction; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.KsqlAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.regex.Pattern; import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.Assert; @Getter @ToString @EqualsAndHashCode public class Permission { private static final List RBAC_ACTION_EXEMPT_LIST = List.of(KSQL, CLUSTERCONFIG, APPLICATIONCONFIG, ACL, AUDIT); Resource resource; List actions; @Nullable String value; @Nullable transient Pattern compiledValuePattern; @SuppressWarnings("unused") public void setResource(String resource) { this.resource = Resource.fromString(resource.toUpperCase()); } @SuppressWarnings("unused") public void setValue(@Nullable String value) { this.value = value; } @SuppressWarnings("unused") public void setActions(List actions) { this.actions = actions; } public void validate() { Assert.notNull(resource, "resource cannot be null"); if (!RBAC_ACTION_EXEMPT_LIST.contains(this.resource)) { Assert.notNull(value, "permission value can't be empty for resource " + resource); } } public void transform() { if (value != null) { this.compiledValuePattern = Pattern.compile(value); } if (CollectionUtils.isNotEmpty(actions) && actions.stream().anyMatch("ALL"::equalsIgnoreCase)) { this.actions = getAllActionValues(); } } private List getAllActionValues() { if (resource == null) { return Collections.emptyList(); } return switch (this.resource) { case APPLICATIONCONFIG -> Arrays.stream(ApplicationConfigAction.values()).map(Enum::toString).toList(); case CLUSTERCONFIG -> Arrays.stream(ClusterConfigAction.values()).map(Enum::toString).toList(); case TOPIC -> Arrays.stream(TopicAction.values()).map(Enum::toString).toList(); case CONSUMER -> Arrays.stream(ConsumerGroupAction.values()).map(Enum::toString).toList(); case SCHEMA -> Arrays.stream(SchemaAction.values()).map(Enum::toString).toList(); case CONNECT -> Arrays.stream(ConnectAction.values()).map(Enum::toString).toList(); case KSQL -> Arrays.stream(KsqlAction.values()).map(Enum::toString).toList(); case ACL -> Arrays.stream(AclAction.values()).map(Enum::toString).toList(); case AUDIT -> Arrays.stream(AuditAction.values()).map(Enum::toString).toList(); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Resource.java ================================================ package com.provectus.kafka.ui.model.rbac; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum Resource { APPLICATIONCONFIG, CLUSTERCONFIG, TOPIC, CONSUMER, SCHEMA, CONNECT, KSQL, ACL, AUDIT; @Nullable public static Resource fromString(String name) { return EnumUtils.getEnum(Resource.class, name); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Role.java ================================================ package com.provectus.kafka.ui.model.rbac; import java.util.List; import lombok.Data; @Data public class Role { String name; List clusters; List subjects; List permissions; public void validate() { permissions.forEach(Permission::transform); permissions.forEach(Permission::validate); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/Subject.java ================================================ package com.provectus.kafka.ui.model.rbac; import com.provectus.kafka.ui.model.rbac.provider.Provider; import lombok.Getter; @Getter public class Subject { Provider provider; String type; String value; public void setProvider(String provider) { this.provider = Provider.fromString(provider.toUpperCase()); } public void setType(String type) { this.type = type; } public void setValue(String value) { this.value = value; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/AclAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum AclAction implements PermissibleAction { VIEW, EDIT ; public static final Set ALTER_ACTIONS = Set.of(EDIT); @Nullable public static AclAction fromString(String name) { return EnumUtils.getEnum(AclAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/ApplicationConfigAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum ApplicationConfigAction implements PermissibleAction { VIEW, EDIT ; public static final Set ALTER_ACTIONS = Set.of(EDIT); @Nullable public static ApplicationConfigAction fromString(String name) { return EnumUtils.getEnum(ApplicationConfigAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/AuditAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum AuditAction implements PermissibleAction { VIEW ; private static final Set ALTER_ACTIONS = Set.of(); @Nullable public static AuditAction fromString(String name) { return EnumUtils.getEnum(AuditAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/ClusterConfigAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum ClusterConfigAction implements PermissibleAction { VIEW, EDIT ; public static final Set ALTER_ACTIONS = Set.of(EDIT); @Nullable public static ClusterConfigAction fromString(String name) { return EnumUtils.getEnum(ClusterConfigAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/ConnectAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum ConnectAction implements PermissibleAction { VIEW, EDIT, CREATE, RESTART ; public static final Set ALTER_ACTIONS = Set.of(CREATE, EDIT, RESTART); @Nullable public static ConnectAction fromString(String name) { return EnumUtils.getEnum(ConnectAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/ConsumerGroupAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum ConsumerGroupAction implements PermissibleAction { VIEW, DELETE, RESET_OFFSETS ; public static final Set ALTER_ACTIONS = Set.of(DELETE, RESET_OFFSETS); @Nullable public static ConsumerGroupAction fromString(String name) { return EnumUtils.getEnum(ConsumerGroupAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/KsqlAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum KsqlAction implements PermissibleAction { EXECUTE ; public static final Set ALTER_ACTIONS = Set.of(EXECUTE); @Nullable public static KsqlAction fromString(String name) { return EnumUtils.getEnum(KsqlAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/PermissibleAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; public sealed interface PermissibleAction permits AclAction, ApplicationConfigAction, ConsumerGroupAction, SchemaAction, ConnectAction, ClusterConfigAction, KsqlAction, TopicAction, AuditAction { String name(); boolean isAlter(); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/SchemaAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum SchemaAction implements PermissibleAction { VIEW, CREATE, DELETE, EDIT, MODIFY_GLOBAL_COMPATIBILITY ; public static final Set ALTER_ACTIONS = Set.of(CREATE, DELETE, EDIT, MODIFY_GLOBAL_COMPATIBILITY); @Nullable public static SchemaAction fromString(String name) { return EnumUtils.getEnum(SchemaAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/permission/TopicAction.java ================================================ package com.provectus.kafka.ui.model.rbac.permission; import java.util.Set; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum TopicAction implements PermissibleAction { VIEW, CREATE, EDIT, DELETE, MESSAGES_READ, MESSAGES_PRODUCE, MESSAGES_DELETE, ; public static final Set ALTER_ACTIONS = Set.of(CREATE, EDIT, DELETE, MESSAGES_PRODUCE, MESSAGES_DELETE); @Nullable public static TopicAction fromString(String name) { return EnumUtils.getEnum(TopicAction.class, name); } @Override public boolean isAlter() { return ALTER_ACTIONS.contains(this); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/rbac/provider/Provider.java ================================================ package com.provectus.kafka.ui.model.rbac.provider; import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; public enum Provider { OAUTH_GOOGLE, OAUTH_GITHUB, OAUTH_COGNITO, OAUTH, LDAP, LDAP_AD; @Nullable public static Provider fromString(String name) { return EnumUtils.getEnum(Provider.class, name); } public static class Name { public static String GOOGLE = "google"; public static String GITHUB = "github"; public static String COGNITO = "cognito"; public static String OAUTH = "oauth"; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/schemaregistry/ErrorResponse.java ================================================ package com.provectus.kafka.ui.model.schemaregistry; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data public class ErrorResponse { @JsonProperty("error_code") private int errorCode; private String message; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/schemaregistry/InternalCompatibilityCheck.java ================================================ package com.provectus.kafka.ui.model.schemaregistry; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data public class InternalCompatibilityCheck { @JsonProperty("is_compatible") private boolean isCompatible; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/schemaregistry/InternalCompatibilityLevel.java ================================================ package com.provectus.kafka.ui.model.schemaregistry; import lombok.Data; @Data public class InternalCompatibilityLevel { private String compatibilityLevel; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/schemaregistry/InternalNewSchema.java ================================================ package com.provectus.kafka.ui.model.schemaregistry; import com.fasterxml.jackson.annotation.JsonInclude; import com.provectus.kafka.ui.model.SchemaTypeDTO; import lombok.Data; @Data public class InternalNewSchema { private String schema; @JsonInclude(JsonInclude.Include.NON_NULL) private SchemaTypeDTO schemaType; public InternalNewSchema(String schema, SchemaTypeDTO schemaType) { this.schema = schema; this.schemaType = schemaType; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/model/schemaregistry/SubjectIdResponse.java ================================================ package com.provectus.kafka.ui.model.schemaregistry; import lombok.Data; @Data public class SubjectIdResponse { private Integer id; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/BuiltInSerde.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.Serde; public interface BuiltInSerde extends Serde { // returns true is serde has enough properties set on cluster&global levels to // be configured without explicit config provide default boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { return true; } // will be called for build-in serdes that were not explicitly registered // and that returned true on canBeAutoConfigured(..) call. // NOTE: Serde.configure() method won't be called if serde is auto-configured! default void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { } @Override default void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ClassloaderUtil.java ================================================ package com.provectus.kafka.ui.serdes; class ClassloaderUtil { static ClassLoader compareAndSwapLoaders(ClassLoader loader) { ClassLoader current = Thread.currentThread().getContextClassLoader(); if (!current.equals(loader)) { Thread.currentThread().setContextClassLoader(loader); } return current; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ClusterSerdes.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import java.io.Closeable; import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor public class ClusterSerdes implements Closeable { final Map serdes; @Nullable final SerdeInstance defaultKeySerde; @Nullable final SerdeInstance defaultValueSerde; @Getter final SerdeInstance fallbackSerde; private Optional findSerdeByPatternsOrDefault(String topic, Serde.Target type, Predicate additionalCheck) { // iterating over serdes in the same order they were added in config for (SerdeInstance serdeInstance : serdes.values()) { var pattern = type == Serde.Target.KEY ? serdeInstance.topicKeyPattern : serdeInstance.topicValuePattern; if (pattern != null && pattern.matcher(topic).matches() && additionalCheck.test(serdeInstance)) { return Optional.of(serdeInstance); } } if (type == Serde.Target.KEY && defaultKeySerde != null && additionalCheck.test(defaultKeySerde)) { return Optional.of(defaultKeySerde); } if (type == Serde.Target.VALUE && defaultValueSerde != null && additionalCheck.test(defaultValueSerde)) { return Optional.of(defaultValueSerde); } return Optional.empty(); } public Optional serdeForName(String name) { return Optional.ofNullable(serdes.get(name)); } public Stream all() { return serdes.values().stream(); } public SerdeInstance suggestSerdeForSerialize(String topic, Serde.Target type) { return findSerdeByPatternsOrDefault(topic, type, s -> s.canSerialize(topic, type)) .orElse(serdes.get(StringSerde.name())); } public SerdeInstance suggestSerdeForDeserialize(String topic, Serde.Target type) { return findSerdeByPatternsOrDefault(topic, type, s -> s.canDeserialize(topic, type)) .orElse(serdes.get(StringSerde.name())); } @Override public void close() { serdes.values().forEach(SerdeInstance::close); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializer.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageDTO.TimestampTypeEnum; import com.provectus.kafka.ui.serde.api.Serde; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.function.UnaryOperator; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.utils.Bytes; @Slf4j @RequiredArgsConstructor public class ConsumerRecordDeserializer { private static final ZoneId UTC_ZONE_ID = ZoneId.of("UTC"); private final String keySerdeName; private final Serde.Deserializer keyDeserializer; private final String valueSerdeName; private final Serde.Deserializer valueDeserializer; private final String fallbackSerdeName; private final Serde.Deserializer fallbackKeyDeserializer; private final Serde.Deserializer fallbackValueDeserializer; private final UnaryOperator masker; public TopicMessageDTO deserialize(ConsumerRecord rec) { var message = new TopicMessageDTO(); fillKey(message, rec); fillValue(message, rec); fillHeaders(message, rec); message.setPartition(rec.partition()); message.setOffset(rec.offset()); message.setTimestampType(mapToTimestampType(rec.timestampType())); message.setTimestamp(OffsetDateTime.ofInstant(Instant.ofEpochMilli(rec.timestamp()), UTC_ZONE_ID)); message.setKeySize(getKeySize(rec)); message.setValueSize(getValueSize(rec)); message.setHeadersSize(getHeadersSize(rec)); return masker.apply(message); } private static TimestampTypeEnum mapToTimestampType(TimestampType timestampType) { return switch (timestampType) { case CREATE_TIME -> TimestampTypeEnum.CREATE_TIME; case LOG_APPEND_TIME -> TimestampTypeEnum.LOG_APPEND_TIME; case NO_TIMESTAMP_TYPE -> TimestampTypeEnum.NO_TIMESTAMP_TYPE; }; } private void fillHeaders(TopicMessageDTO message, ConsumerRecord rec) { Map headers = new HashMap<>(); rec.headers().iterator() .forEachRemaining(header -> headers.put( header.key(), header.value() != null ? new String(header.value()) : null )); message.setHeaders(headers); } private void fillKey(TopicMessageDTO message, ConsumerRecord rec) { if (rec.key() == null) { return; } try { var deserResult = keyDeserializer.deserialize(new RecordHeadersImpl(), rec.key().get()); message.setKey(deserResult.getResult()); message.setKeySerde(keySerdeName); message.setKeyDeserializeProperties(deserResult.getAdditionalProperties()); } catch (Exception e) { log.trace("Error deserializing key for key topic: {}, partition {}, offset {}, with serde {}", rec.topic(), rec.partition(), rec.offset(), keySerdeName, e); var deserResult = fallbackKeyDeserializer.deserialize(new RecordHeadersImpl(), rec.key().get()); message.setKey(deserResult.getResult()); message.setKeySerde(fallbackSerdeName); } } private void fillValue(TopicMessageDTO message, ConsumerRecord rec) { if (rec.value() == null) { return; } try { var deserResult = valueDeserializer.deserialize( new RecordHeadersImpl(rec.headers()), rec.value().get()); message.setContent(deserResult.getResult()); message.setValueSerde(valueSerdeName); message.setValueDeserializeProperties(deserResult.getAdditionalProperties()); } catch (Exception e) { log.trace("Error deserializing key for value topic: {}, partition {}, offset {}, with serde {}", rec.topic(), rec.partition(), rec.offset(), valueSerdeName, e); var deserResult = fallbackValueDeserializer.deserialize( new RecordHeadersImpl(rec.headers()), rec.value().get()); message.setContent(deserResult.getResult()); message.setValueSerde(fallbackSerdeName); } } private static Long getHeadersSize(ConsumerRecord consumerRecord) { Headers headers = consumerRecord.headers(); if (headers != null) { return Arrays.stream(headers.toArray()) .mapToLong(ConsumerRecordDeserializer::headerSize) .sum(); } return 0L; } private static Long getKeySize(ConsumerRecord consumerRecord) { return consumerRecord.key() != null ? (long) consumerRecord.serializedKeySize() : null; } private static Long getValueSize(ConsumerRecord consumerRecord) { return consumerRecord.value() != null ? (long) consumerRecord.serializedValueSize() : null; } private static int headerSize(Header header) { int key = header.key() != null ? header.key().getBytes().length : 0; int val = header.value() != null ? header.value().length : 0; return key + val; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/CustomSerdeLoader.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.Serde; import java.io.IOException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Enumeration; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.Value; class CustomSerdeLoader { @Value static class CustomSerde { Serde serde; ClassLoader classLoader; } // serde location -> classloader private final Map classloaders = new ConcurrentHashMap<>(); @SneakyThrows CustomSerde loadAndConfigure(String className, String filePath, PropertyResolver serdeProps, PropertyResolver clusterProps, PropertyResolver globalProps) { Path locationPath = Path.of(filePath); var serdeClassloader = createClassloader(locationPath); var origCL = ClassloaderUtil.compareAndSwapLoaders(serdeClassloader); try { var serdeClass = serdeClassloader.loadClass(className); var serde = (Serde) serdeClass.getDeclaredConstructor().newInstance(); serde.configure(serdeProps, clusterProps, globalProps); return new CustomSerde(serde, serdeClassloader); } finally { ClassloaderUtil.compareAndSwapLoaders(origCL); } } private static boolean isArchive(Path path) { String archivePath = path.toString().toLowerCase(); return Files.isReadable(path) && Files.isRegularFile(path) && (archivePath.endsWith(".jar") || archivePath.endsWith(".zip")); } @SneakyThrows private static List findArchiveFiles(Path location) { if (isArchive(location)) { return List.of(location.toUri().toURL()); } if (Files.isDirectory(location)) { List archiveFiles = new ArrayList<>(); try (var files = Files.walk(location)) { var paths = files.filter(CustomSerdeLoader::isArchive).collect(Collectors.toList()); for (Path path : paths) { archiveFiles.add(path.toUri().toURL()); } } return archiveFiles; } return List.of(); } private ClassLoader createClassloader(Path location) { if (!Files.exists(location)) { throw new IllegalStateException("Location does not exist"); } var archives = findArchiveFiles(location); if (archives.isEmpty()) { throw new IllegalStateException("No archive files were found"); } // we assume that location's content does not change during serdes creation // so, we can reuse already created classloaders return classloaders.computeIfAbsent(location, l -> AccessController.doPrivileged( (PrivilegedAction) () -> new ChildFirstClassloader( archives.toArray(URL[]::new), CustomSerdeLoader.class.getClassLoader()))); } //--------------------------------------------------------------------------------- // This Classloader first tries to load classes by itself. If class not fount // search is propagated to parent (this is opposite to how usual classloaders work) private static class ChildFirstClassloader extends URLClassLoader { private static final String JAVA_PACKAGE_PREFIX = "java."; ChildFirstClassloader(URL[] urls, ClassLoader parent) { super(urls, parent); } @Override protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { // first check whether it's a system class, delegate to the system loader if (name.startsWith(JAVA_PACKAGE_PREFIX)) { return findSystemClass(name); } Class loadedClass = findLoadedClass(name); if (loadedClass == null) { try { // start searching from current classloader loadedClass = findClass(name); } catch (ClassNotFoundException e) { // if not found - going to parent loadedClass = super.loadClass(name, resolve); } } if (resolve) { resolveClass(loadedClass); } return loadedClass; } @Override public Enumeration getResources(String name) throws IOException { List allRes = new LinkedList<>(); Enumeration thisRes = findResources(name); if (thisRes != null) { while (thisRes.hasMoreElements()) { allRes.add(thisRes.nextElement()); } } // then try finding resources from parent classloaders Enumeration parentRes = super.findResources(name); if (parentRes != null) { while (parentRes.hasMoreElements()) { allRes.add(parentRes.nextElement()); } } return new Enumeration<>() { final Iterator it = allRes.iterator(); @Override public boolean hasMoreElements() { return it.hasNext(); } @Override public URL nextElement() { return it.next(); } }; } @Override public URL getResource(String name) { URL res = findResource(name); if (res == null) { res = super.getResource(name); } return res; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/ProducerRecordCreator.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.Serde; import java.util.Map; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; @RequiredArgsConstructor public class ProducerRecordCreator { private final Serde.Serializer keySerializer; private final Serde.Serializer valuesSerializer; public ProducerRecord create(String topic, @Nullable Integer partition, @Nullable String key, @Nullable String value, @Nullable Map headers) { return new ProducerRecord<>( topic, partition, key == null ? null : keySerializer.serialize(key), value == null ? null : valuesSerializer.serialize(value), headers == null ? null : createHeaders(headers) ); } private Iterable
createHeaders(Map clientHeaders) { RecordHeaders headers = new RecordHeaders(); clientHeaders.forEach((k, v) -> headers.add(new RecordHeader(k, v.getBytes()))); return headers; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/PropertyResolverImpl.java ================================================ package com.provectus.kafka.ui.serdes; import com.google.common.base.Preconditions; import com.provectus.kafka.ui.serde.api.PropertyResolver; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.Nullable; import org.springframework.boot.context.properties.bind.Bindable; import org.springframework.boot.context.properties.bind.Binder; import org.springframework.boot.context.properties.source.ConfigurationPropertyName; import org.springframework.core.env.Environment; import org.springframework.core.env.StandardEnvironment; public class PropertyResolverImpl implements PropertyResolver { private final Binder binder; @Nullable private final String prefix; public static PropertyResolverImpl empty() { return new PropertyResolverImpl(new StandardEnvironment(), null); } public PropertyResolverImpl(Environment env) { this(env, null); } public PropertyResolverImpl(Environment env, @Nullable String prefix) { this.binder = Binder.get(env); this.prefix = prefix; } private ConfigurationPropertyName targetPropertyName(String key) { Preconditions.checkNotNull(key); Preconditions.checkState(!key.isBlank()); String propertyName = prefix == null ? key : prefix + "." + key; return ConfigurationPropertyName.adapt(propertyName, '.'); } @Override public Optional getProperty(String key, Class targetType) { var targetKey = targetPropertyName(key); var result = binder.bind(targetKey, Bindable.of(targetType)); return result.isBound() ? Optional.of(result.get()) : Optional.empty(); } @Override public Optional> getListProperty(String key, Class itemType) { var targetKey = targetPropertyName(key); var listResult = binder.bind(targetKey, Bindable.listOf(itemType)); return listResult.isBound() ? Optional.of(listResult.get()) : Optional.empty(); } @Override public Optional> getMapProperty(String key, Class keyType, Class valueType) { var targetKey = targetPropertyName(key); var mapResult = binder.bind(targetKey, Bindable.mapOf(keyType, valueType)); return mapResult.isBound() ? Optional.of(mapResult.get()) : Optional.empty(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/RecordHeaderImpl.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.RecordHeader; import org.apache.kafka.common.header.Header; public class RecordHeaderImpl implements RecordHeader { private final Header header; public RecordHeaderImpl(Header header) { this.header = header; } @Override public String key() { return header.key(); } @Override public byte[] value() { return header.value(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/RecordHeadersImpl.java ================================================ package com.provectus.kafka.ui.serdes; import com.google.common.collect.Iterators; import com.provectus.kafka.ui.serde.api.RecordHeader; import com.provectus.kafka.ui.serde.api.RecordHeaders; import java.util.Iterator; import org.apache.kafka.common.header.Headers; public class RecordHeadersImpl implements RecordHeaders { private final Headers headers; public RecordHeadersImpl() { this(new org.apache.kafka.common.header.internals.RecordHeaders()); } public RecordHeadersImpl(Headers headers) { this.headers = headers; } @Override public Iterator iterator() { return Iterators.transform(headers.iterator(), RecordHeaderImpl::new); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdeInstance.java ================================================ package com.provectus.kafka.ui.serdes; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serde.api.Serde; import java.io.Closeable; import java.util.Optional; import java.util.function.Supplier; import java.util.regex.Pattern; import javax.annotation.Nullable; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor public class SerdeInstance implements Closeable { @Getter final String name; final Serde serde; @Nullable final Pattern topicKeyPattern; @Nullable final Pattern topicValuePattern; @Nullable // will be set for custom serdes final ClassLoader classLoader; private T wrapWithClassloader(Supplier call) { if (classLoader == null) { return call.get(); } var origCl = ClassloaderUtil.compareAndSwapLoaders(classLoader); try { return call.get(); } finally { ClassloaderUtil.compareAndSwapLoaders(origCl); } } public Optional getSchema(String topic, Serde.Target type) { try { return wrapWithClassloader(() -> serde.getSchema(topic, type)); } catch (Exception e) { log.warn("Error getting schema for '{}'({}) with serde '{}'", topic, type, name, e); return Optional.empty(); } } public Optional description() { try { return wrapWithClassloader(serde::getDescription); } catch (Exception e) { log.warn("Error getting description serde '{}'", name, e); return Optional.empty(); } } public boolean canSerialize(String topic, Serde.Target type) { try { return wrapWithClassloader(() -> serde.canSerialize(topic, type)); } catch (Exception e) { log.warn("Error calling canSerialize for '{}'({}) with serde '{}'", topic, type, name, e); return false; } } public boolean canDeserialize(String topic, Serde.Target type) { try { return wrapWithClassloader(() -> serde.canDeserialize(topic, type)); } catch (Exception e) { log.warn("Error calling canDeserialize for '{}'({}) with serde '{}'", topic, type, name, e); return false; } } public Serde.Serializer serializer(String topic, Serde.Target type) { return wrapWithClassloader(() -> { var serializer = serde.serializer(topic, type); return input -> wrapWithClassloader(() -> serializer.serialize(input)); }); } public Serde.Deserializer deserializer(String topic, Serde.Target type) { return wrapWithClassloader(() -> { var deserializer = serde.deserializer(topic, type); return (headers, data) -> wrapWithClassloader(() -> deserializer.deserialize(headers, data)); }); } @Override public void close() { wrapWithClassloader(() -> { try { serde.close(); } catch (Exception e) { log.error("Error closing serde " + name, e); } return null; }); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/SerdesInitializer.java ================================================ package com.provectus.kafka.ui.serdes; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.config.ClustersProperties.SerdeConfig; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.builtin.AvroEmbeddedSerde; import com.provectus.kafka.ui.serdes.builtin.Base64Serde; import com.provectus.kafka.ui.serdes.builtin.ConsumerOffsetsSerde; import com.provectus.kafka.ui.serdes.builtin.HexSerde; import com.provectus.kafka.ui.serdes.builtin.Int32Serde; import com.provectus.kafka.ui.serdes.builtin.Int64Serde; import com.provectus.kafka.ui.serdes.builtin.ProtobufFileSerde; import com.provectus.kafka.ui.serdes.builtin.ProtobufRawSerde; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import com.provectus.kafka.ui.serdes.builtin.UInt32Serde; import com.provectus.kafka.ui.serdes.builtin.UInt64Serde; import com.provectus.kafka.ui.serdes.builtin.UuidBinarySerde; import com.provectus.kafka.ui.serdes.builtin.sr.SchemaRegistrySerde; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import javax.annotation.Nullable; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.core.env.Environment; @Slf4j public class SerdesInitializer { private final Map> builtInSerdeClasses; private final CustomSerdeLoader customSerdeLoader; public SerdesInitializer() { this( ImmutableMap.>builder() .put(StringSerde.name(), StringSerde.class) .put(SchemaRegistrySerde.name(), SchemaRegistrySerde.class) .put(ProtobufFileSerde.name(), ProtobufFileSerde.class) .put(Int32Serde.name(), Int32Serde.class) .put(Int64Serde.name(), Int64Serde.class) .put(UInt32Serde.name(), UInt32Serde.class) .put(UInt64Serde.name(), UInt64Serde.class) .put(AvroEmbeddedSerde.name(), AvroEmbeddedSerde.class) .put(Base64Serde.name(), Base64Serde.class) .put(HexSerde.name(), HexSerde.class) .put(UuidBinarySerde.name(), UuidBinarySerde.class) .put(ProtobufRawSerde.name(), ProtobufRawSerde.class) .build(), new CustomSerdeLoader() ); } @VisibleForTesting SerdesInitializer(Map> builtInSerdeClasses, CustomSerdeLoader customSerdeLoader) { this.builtInSerdeClasses = builtInSerdeClasses; this.customSerdeLoader = customSerdeLoader; } /** * Initialization algorithm: * First, we iterate over explicitly configured serdes from cluster config: * > if serde has name = one of built-in serde's names: * - if serde's properties are empty, we treat it as serde should be * auto-configured - we try to do that * - if serde's properties not empty, we treat it as an intention to * override default configuration, so we configuring it with specific config (calling configure(..)) *

* > if serde has className = one of built-in serde's classes: * - initializing it with specific config and with default classloader *

* > if serde has custom className != one of built-in serde's classes: * - initializing it with specific config and with custom classloader (see CustomSerdeLoader) *

* Second, we iterate over remaining built-in serdes (that we NOT explicitly configured by config) * trying to auto-configure them and registering with empty patterns - they will be present * in Serde selection in UI, but not assigned to any topic k/v. */ public ClusterSerdes init(Environment env, ClustersProperties clustersProperties, int clusterIndex) { ClustersProperties.Cluster clusterProperties = clustersProperties.getClusters().get(clusterIndex); log.debug("Configuring serdes for cluster {}", clusterProperties.getName()); var globalPropertiesResolver = new PropertyResolverImpl(env); var clusterPropertiesResolver = new PropertyResolverImpl(env, "kafka.clusters." + clusterIndex); Map registeredSerdes = new LinkedHashMap<>(); // initializing serdes from config if (clusterProperties.getSerde() != null) { for (int i = 0; i < clusterProperties.getSerde().size(); i++) { SerdeConfig serdeConfig = clusterProperties.getSerde().get(i); if (Strings.isNullOrEmpty(serdeConfig.getName())) { throw new ValidationException("'name' property not set for serde: " + serdeConfig); } if (registeredSerdes.containsKey(serdeConfig.getName())) { throw new ValidationException("Multiple serdes with same name: " + serdeConfig.getName()); } var instance = createSerdeFromConfig( serdeConfig, new PropertyResolverImpl(env, "kafka.clusters." + clusterIndex + ".serde." + i + ".properties"), clusterPropertiesResolver, globalPropertiesResolver ); registeredSerdes.put(serdeConfig.getName(), instance); } } // initializing remaining built-in serdes with empty selection patters builtInSerdeClasses.forEach((name, clazz) -> { if (!registeredSerdes.containsKey(name)) { BuiltInSerde serde = createSerdeInstance(clazz); if (autoConfigureSerde(serde, clusterPropertiesResolver, globalPropertiesResolver)) { registeredSerdes.put(name, new SerdeInstance(name, serde, null, null, null)); } } }); registerTopicRelatedSerde(registeredSerdes); return new ClusterSerdes( registeredSerdes, Optional.ofNullable(clusterProperties.getDefaultKeySerde()) .map(name -> Preconditions.checkNotNull(registeredSerdes.get(name), "Default key serde not found")) .orElse(null), Optional.ofNullable(clusterProperties.getDefaultValueSerde()) .map(name -> Preconditions.checkNotNull(registeredSerdes.get(name), "Default value serde not found")) .or(() -> Optional.ofNullable(registeredSerdes.get(SchemaRegistrySerde.name()))) .or(() -> Optional.ofNullable(registeredSerdes.get(ProtobufFileSerde.name()))) .orElse(null), createFallbackSerde() ); } /** * Registers serdse that should only be used for specific (hard-coded) topics, like ConsumerOffsetsSerde. */ private void registerTopicRelatedSerde(Map serdes) { registerConsumerOffsetsSerde(serdes); } private void registerConsumerOffsetsSerde(Map serdes) { var pattern = Pattern.compile(ConsumerOffsetsSerde.TOPIC); serdes.put( ConsumerOffsetsSerde.name(), new SerdeInstance( ConsumerOffsetsSerde.name(), new ConsumerOffsetsSerde(), pattern, pattern, null ) ); } private SerdeInstance createFallbackSerde() { StringSerde serde = new StringSerde(); serde.configure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); return new SerdeInstance("Fallback", serde, null, null, null); } @SneakyThrows private SerdeInstance createSerdeFromConfig(SerdeConfig serdeConfig, PropertyResolver serdeProps, PropertyResolver clusterProps, PropertyResolver globalProps) { if (builtInSerdeClasses.containsKey(serdeConfig.getName())) { return createSerdeWithBuiltInSerdeName(serdeConfig, serdeProps, clusterProps, globalProps); } if (serdeConfig.getClassName() != null) { var builtInSerdeClass = builtInSerdeClasses.values().stream() .filter(c -> c.getName().equals(serdeConfig.getClassName())) .findAny(); // built-in serde type with custom name if (builtInSerdeClass.isPresent()) { return createSerdeWithBuiltInClass(builtInSerdeClass.get(), serdeConfig, serdeProps, clusterProps, globalProps); } } log.info("Loading custom serde {}", serdeConfig.getName()); return loadAndInitCustomSerde(serdeConfig, serdeProps, clusterProps, globalProps); } private SerdeInstance createSerdeWithBuiltInSerdeName(SerdeConfig serdeConfig, PropertyResolver serdeProps, PropertyResolver clusterProps, PropertyResolver globalProps) { String name = serdeConfig.getName(); if (serdeConfig.getClassName() != null) { throw new ValidationException("className can't be set for built-in serde"); } if (serdeConfig.getFilePath() != null) { throw new ValidationException("filePath can't be set for built-in serde types"); } var clazz = builtInSerdeClasses.get(name); BuiltInSerde serde = createSerdeInstance(clazz); if (serdeConfig.getProperties() == null || serdeConfig.getProperties().isEmpty()) { if (!autoConfigureSerde(serde, clusterProps, globalProps)) { // no properties provided and serde does not support auto-configuration throw new ValidationException(name + " serde is not configured"); } } else { // configuring serde with explicitly set properties serde.configure(serdeProps, clusterProps, globalProps); } return new SerdeInstance( name, serde, nullablePattern(serdeConfig.getTopicKeysPattern()), nullablePattern(serdeConfig.getTopicValuesPattern()), null ); } private boolean autoConfigureSerde(BuiltInSerde serde, PropertyResolver clusterProps, PropertyResolver globalProps) { if (serde.canBeAutoConfigured(clusterProps, globalProps)) { serde.autoConfigure(clusterProps, globalProps); return true; } return false; } @SneakyThrows private SerdeInstance createSerdeWithBuiltInClass(Class clazz, SerdeConfig serdeConfig, PropertyResolver serdeProps, PropertyResolver clusterProps, PropertyResolver globalProps) { if (serdeConfig.getFilePath() != null) { throw new ValidationException("filePath can't be set for built-in serde type"); } BuiltInSerde serde = createSerdeInstance(clazz); serde.configure(serdeProps, clusterProps, globalProps); return new SerdeInstance( serdeConfig.getName(), serde, nullablePattern(serdeConfig.getTopicKeysPattern()), nullablePattern(serdeConfig.getTopicValuesPattern()), null ); } @SneakyThrows private T createSerdeInstance(Class clazz) { return clazz.getDeclaredConstructor().newInstance(); } private SerdeInstance loadAndInitCustomSerde(SerdeConfig serdeConfig, PropertyResolver serdeProps, PropertyResolver clusterProps, PropertyResolver globalProps) { if (Strings.isNullOrEmpty(serdeConfig.getClassName())) { throw new ValidationException( "'className' property not set for custom serde " + serdeConfig.getName()); } if (Strings.isNullOrEmpty(serdeConfig.getFilePath())) { throw new ValidationException( "'filePath' property not set for custom serde " + serdeConfig.getName()); } var loaded = customSerdeLoader.loadAndConfigure( serdeConfig.getClassName(), serdeConfig.getFilePath(), serdeProps, clusterProps, globalProps); return new SerdeInstance( serdeConfig.getName(), loaded.getSerde(), nullablePattern(serdeConfig.getTopicKeysPattern()), nullablePattern(serdeConfig.getTopicValuesPattern()), loaded.getClassLoader() ); } @Nullable private Pattern nullablePattern(@Nullable String pattern) { return pattern == null ? null : Pattern.compile(pattern); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/AvroEmbeddedSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.RecordHeaders; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import io.confluent.kafka.schemaregistry.avro.AvroSchemaUtils; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import org.apache.avro.file.DataFileReader; import org.apache.avro.file.SeekableByteArrayInput; import org.apache.avro.generic.GenericDatumReader; public class AvroEmbeddedSerde implements BuiltInSerde { public static String name() { return "Avro (Embedded)"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return false; } @Override public Serializer serializer(String topic, Target type) { throw new IllegalStateException(); } @Override public Deserializer deserializer(String topic, Target type) { return new Deserializer() { @SneakyThrows @Override public DeserializeResult deserialize(RecordHeaders headers, byte[] data) { try (var reader = new DataFileReader<>(new SeekableByteArrayInput(data), new GenericDatumReader<>())) { if (!reader.hasNext()) { // this is very strange situation, when only header present in payload // returning null in this case return new DeserializeResult(null, DeserializeResult.Type.JSON, Map.of()); } Object avroObj = reader.next(); String jsonValue = new String(AvroSchemaUtils.toJson(avroObj)); return new DeserializeResult(jsonValue, DeserializeResult.Type.JSON, Map.of()); } } }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/Base64Serde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Base64; import java.util.Map; import java.util.Optional; public class Base64Serde implements BuiltInSerde { public static String name() { return "Base64"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { var decoder = Base64.getDecoder(); return inputString -> { inputString = inputString.trim(); // it is actually a hack to provide ability to sent empty array as a key/value if (inputString.length() == 0) { return new byte[] {}; } return decoder.decode(inputString); }; } @Override public Deserializer deserializer(String topic, Target type) { var encoder = Base64.getEncoder(); return (headers, data) -> new DeserializeResult( encoder.encodeToString(data), DeserializeResult.Type.STRING, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/ConsumerOffsetsSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.BoundField; import org.apache.kafka.common.protocol.types.CompactArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.protocol.types.Type; // Deserialization logic and message's schemas can be found in // kafka.coordinator.group.GroupMetadataManager (readMessageKey, readOffsetMessageValue, readGroupMessageValue) public class ConsumerOffsetsSerde implements BuiltInSerde { private static final JsonMapper JSON_MAPPER = createMapper(); private static final String ASSIGNMENT = "assignment"; private static final String CLIENT_HOST = "client_host"; private static final String CLIENT_ID = "client_id"; private static final String COMMIT_TIMESTAMP = "commit_timestamp"; private static final String CURRENT_STATE_TIMESTAMP = "current_state_timestamp"; private static final String GENERATION = "generation"; private static final String LEADER = "leader"; private static final String MEMBERS = "members"; private static final String MEMBER_ID = "member_id"; private static final String METADATA = "metadata"; private static final String OFFSET = "offset"; private static final String PROTOCOL = "protocol"; private static final String PROTOCOL_TYPE = "protocol_type"; private static final String REBALANCE_TIMEOUT = "rebalance_timeout"; private static final String SESSION_TIMEOUT = "session_timeout"; private static final String SUBSCRIPTION = "subscription"; public static final String TOPIC = "__consumer_offsets"; public static String name() { return "__consumer_offsets"; } private static JsonMapper createMapper() { var module = new SimpleModule(); module.addSerializer(Struct.class, new JsonSerializer<>() { @Override public void serialize(Struct value, JsonGenerator gen, SerializerProvider serializers) throws IOException { gen.writeStartObject(); for (BoundField field : value.schema().fields()) { var fieldVal = value.get(field); gen.writeObjectField(field.def.name, fieldVal); } gen.writeEndObject(); } }); var mapper = new JsonMapper(); mapper.registerModule(module); return mapper; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return topic.equals(TOPIC); } @Override public boolean canSerialize(String topic, Target type) { return false; } @Override public Serializer serializer(String topic, Target type) { throw new UnsupportedOperationException(); } @Override public Deserializer deserializer(String topic, Target type) { return switch (type) { case KEY -> keyDeserializer(); case VALUE -> valueDeserializer(); }; } private Deserializer keyDeserializer() { final Schema commitKeySchema = new Schema( new Field("group", Type.STRING, ""), new Field("topic", Type.STRING, ""), new Field("partition", Type.INT32, "") ); final Schema groupMetadataSchema = new Schema( new Field("group", Type.STRING, "") ); return (headers, data) -> { var bb = ByteBuffer.wrap(data); short version = bb.getShort(); return new DeserializeResult( toJson( switch (version) { case 0, 1 -> commitKeySchema.read(bb); case 2 -> groupMetadataSchema.read(bb); default -> throw new IllegalStateException("Unknown group metadata message version: " + version); } ), DeserializeResult.Type.JSON, Map.of() ); }; } private Deserializer valueDeserializer() { final Schema commitOffsetSchemaV0 = new Schema( new Field(OFFSET, Type.INT64, ""), new Field(METADATA, Type.STRING, ""), new Field(COMMIT_TIMESTAMP, Type.INT64, "") ); final Schema commitOffsetSchemaV1 = new Schema( new Field(OFFSET, Type.INT64, ""), new Field(METADATA, Type.STRING, ""), new Field(COMMIT_TIMESTAMP, Type.INT64, ""), new Field("expire_timestamp", Type.INT64, "") ); final Schema commitOffsetSchemaV2 = new Schema( new Field(OFFSET, Type.INT64, ""), new Field(METADATA, Type.STRING, ""), new Field(COMMIT_TIMESTAMP, Type.INT64, "") ); final Schema commitOffsetSchemaV3 = new Schema( new Field(OFFSET, Type.INT64, ""), new Field("leader_epoch", Type.INT32, ""), new Field(METADATA, Type.STRING, ""), new Field(COMMIT_TIMESTAMP, Type.INT64, "") ); final Schema commitOffsetSchemaV4 = new Schema( new Field(OFFSET, Type.INT64, ""), new Field("leader_epoch", Type.INT32, ""), new Field(METADATA, Type.COMPACT_STRING, ""), new Field(COMMIT_TIMESTAMP, Type.INT64, ""), Field.TaggedFieldsSection.of() ); final Schema metadataSchema0 = new Schema( new Field(PROTOCOL_TYPE, Type.STRING, ""), new Field(GENERATION, Type.INT32, ""), new Field(PROTOCOL, Type.NULLABLE_STRING, ""), new Field(LEADER, Type.NULLABLE_STRING, ""), new Field(MEMBERS, new ArrayOf(new Schema( new Field(MEMBER_ID, Type.STRING, ""), new Field(CLIENT_ID, Type.STRING, ""), new Field(CLIENT_HOST, Type.STRING, ""), new Field(SESSION_TIMEOUT, Type.INT32, ""), new Field(SUBSCRIPTION, Type.BYTES, ""), new Field(ASSIGNMENT, Type.BYTES, "") )), "") ); final Schema metadataSchema1 = new Schema( new Field(PROTOCOL_TYPE, Type.STRING, ""), new Field(GENERATION, Type.INT32, ""), new Field(PROTOCOL, Type.NULLABLE_STRING, ""), new Field(LEADER, Type.NULLABLE_STRING, ""), new Field(MEMBERS, new ArrayOf(new Schema( new Field(MEMBER_ID, Type.STRING, ""), new Field(CLIENT_ID, Type.STRING, ""), new Field(CLIENT_HOST, Type.STRING, ""), new Field(REBALANCE_TIMEOUT, Type.INT32, ""), new Field(SESSION_TIMEOUT, Type.INT32, ""), new Field(SUBSCRIPTION, Type.BYTES, ""), new Field(ASSIGNMENT, Type.BYTES, "") )), "") ); final Schema metadataSchema2 = new Schema( new Field(PROTOCOL_TYPE, Type.STRING, ""), new Field(GENERATION, Type.INT32, ""), new Field(PROTOCOL, Type.NULLABLE_STRING, ""), new Field(LEADER, Type.NULLABLE_STRING, ""), new Field(CURRENT_STATE_TIMESTAMP, Type.INT64, ""), new Field(MEMBERS, new ArrayOf(new Schema( new Field(MEMBER_ID, Type.STRING, ""), new Field(CLIENT_ID, Type.STRING, ""), new Field(CLIENT_HOST, Type.STRING, ""), new Field(REBALANCE_TIMEOUT, Type.INT32, ""), new Field(SESSION_TIMEOUT, Type.INT32, ""), new Field(SUBSCRIPTION, Type.BYTES, ""), new Field(ASSIGNMENT, Type.BYTES, "") )), "") ); final Schema metadataSchema3 = new Schema( new Field(PROTOCOL_TYPE, Type.STRING, ""), new Field(GENERATION, Type.INT32, ""), new Field(PROTOCOL, Type.NULLABLE_STRING, ""), new Field(LEADER, Type.NULLABLE_STRING, ""), new Field(CURRENT_STATE_TIMESTAMP, Type.INT64, ""), new Field(MEMBERS, new ArrayOf(new Schema( new Field(MEMBER_ID, Type.STRING, ""), new Field("group_instance_id", Type.NULLABLE_STRING, ""), new Field(CLIENT_ID, Type.STRING, ""), new Field(CLIENT_HOST, Type.STRING, ""), new Field(REBALANCE_TIMEOUT, Type.INT32, ""), new Field(SESSION_TIMEOUT, Type.INT32, ""), new Field(SUBSCRIPTION, Type.BYTES, ""), new Field(ASSIGNMENT, Type.BYTES, "") )), "") ); final Schema metadataSchema4 = new Schema( new Field(PROTOCOL_TYPE, Type.COMPACT_STRING, ""), new Field(GENERATION, Type.INT32, ""), new Field(PROTOCOL, Type.COMPACT_NULLABLE_STRING, ""), new Field(LEADER, Type.COMPACT_NULLABLE_STRING, ""), new Field(CURRENT_STATE_TIMESTAMP, Type.INT64, ""), new Field(MEMBERS, new CompactArrayOf(new Schema( new Field(MEMBER_ID, Type.COMPACT_STRING, ""), new Field("group_instance_id", Type.COMPACT_NULLABLE_STRING, ""), new Field(CLIENT_ID, Type.COMPACT_STRING, ""), new Field(CLIENT_HOST, Type.COMPACT_STRING, ""), new Field(REBALANCE_TIMEOUT, Type.INT32, ""), new Field(SESSION_TIMEOUT, Type.INT32, ""), new Field(SUBSCRIPTION, Type.COMPACT_BYTES, ""), new Field(ASSIGNMENT, Type.COMPACT_BYTES, ""), Field.TaggedFieldsSection.of() )), ""), Field.TaggedFieldsSection.of() ); return (headers, data) -> { String result; var bb = ByteBuffer.wrap(data); short version = bb.getShort(); // ideally, we should distinguish if value is commit or metadata // by checking record's key, but our current serde structure doesn't allow that. // so, we are trying to parse into metadata first and after into commit msg try { result = toJson( switch (version) { case 0 -> metadataSchema0.read(bb); case 1 -> metadataSchema1.read(bb); case 2 -> metadataSchema2.read(bb); case 3 -> metadataSchema3.read(bb); case 4 -> metadataSchema4.read(bb); default -> throw new IllegalArgumentException("Unrecognized version: " + version); } ); } catch (Throwable e) { bb = bb.rewind(); bb.getShort(); // skipping version result = toJson( switch (version) { case 0 -> commitOffsetSchemaV0.read(bb); case 1 -> commitOffsetSchemaV1.read(bb); case 2 -> commitOffsetSchemaV2.read(bb); case 3 -> commitOffsetSchemaV3.read(bb); case 4 -> commitOffsetSchemaV4.read(bb); default -> throw new IllegalArgumentException("Unrecognized version: " + version); } ); } if (bb.remaining() != 0) { throw new IllegalArgumentException( "Message buffer is not read to the end, which is likely means message is unrecognized"); } return new DeserializeResult( result, DeserializeResult.Type.JSON, Map.of() ); }; } @SneakyThrows private String toJson(Struct s) { return JSON_MAPPER.writeValueAsString(s); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/HexSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.HexFormat; import java.util.Map; import java.util.Optional; public class HexSerde implements BuiltInSerde { private HexFormat deserializeHexFormat; public static String name() { return "Hex"; } @Override public void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { configure(" ", true); } @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { String delim = serdeProperties.getProperty("delimiter", String.class).orElse(" "); boolean uppercase = serdeProperties.getProperty("uppercase", Boolean.class).orElse(true); configure(delim, uppercase); } private void configure(String delim, boolean uppercase) { deserializeHexFormat = HexFormat.ofDelimiter(delim); if (uppercase) { deserializeHexFormat = deserializeHexFormat.withUpperCase(); } } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> { input = input.trim(); // it is a hack to provide ability to sent empty array as a key/value if (input.length() == 0) { return new byte[] {}; } return HexFormat.of().parseHex(prepareInputForParse(input)); }; } // removing most-common delimiters and prefixes private static String prepareInputForParse(String input) { return input .replaceAll(" ", "") .replaceAll("#", "") .replaceAll(":", ""); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( deserializeHexFormat.formatHex(data), DeserializeResult.Type.STRING, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/Int32Serde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.common.primitives.Ints; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Map; import java.util.Optional; public class Int32Serde implements BuiltInSerde { public static String name() { return "Int32"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.of( new SchemaDescription( String.format( "{ " + " \"type\" : \"integer\", " + " \"minimum\" : %s, " + " \"maximum\" : %s " + "}", Integer.MIN_VALUE, Integer.MAX_VALUE ), Map.of() ) ); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> Ints.toByteArray(Integer.parseInt(input)); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( String.valueOf(Ints.fromByteArray(data)), DeserializeResult.Type.JSON, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/Int64Serde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.common.primitives.Longs; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.RecordHeaders; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Map; import java.util.Optional; public class Int64Serde implements BuiltInSerde { public static String name() { return "Int64"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.of( new SchemaDescription( String.format( "{ " + " \"type\" : \"integer\", " + " \"minimum\" : %s, " + " \"maximum\" : %s " + "}", Long.MIN_VALUE, Long.MAX_VALUE ), Map.of() ) ); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> Longs.toByteArray(Long.parseLong(input)); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( String.valueOf(Longs.fromByteArray(data)), DeserializeResult.Type.JSON, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/ProtobufFileSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.protobuf.AnyProto; import com.google.protobuf.ApiProto; import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.DurationProto; import com.google.protobuf.DynamicMessage; import com.google.protobuf.EmptyProto; import com.google.protobuf.FieldMaskProto; import com.google.protobuf.SourceContextProto; import com.google.protobuf.StructProto; import com.google.protobuf.TimestampProto; import com.google.protobuf.TypeProto; import com.google.protobuf.WrappersProto; import com.google.protobuf.util.JsonFormat; import com.google.type.ColorProto; import com.google.type.DateProto; import com.google.type.DateTimeProto; import com.google.type.DayOfWeekProto; import com.google.type.ExprProto; import com.google.type.FractionProto; import com.google.type.IntervalProto; import com.google.type.LatLngProto; import com.google.type.MoneyProto; import com.google.type.MonthProto; import com.google.type.PhoneNumberProto; import com.google.type.PostalAddressProto; import com.google.type.QuaternionProto; import com.google.type.TimeOfDayProto; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.RecordHeaders; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import com.provectus.kafka.ui.util.jsonschema.ProtobufSchemaConverter; import com.squareup.wire.schema.ErrorCollector; import com.squareup.wire.schema.Linker; import com.squareup.wire.schema.Loader; import com.squareup.wire.schema.Location; import com.squareup.wire.schema.ProtoFile; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import com.squareup.wire.schema.internal.parser.ProtoParser; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaUtils; import java.io.ByteArrayInputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @Slf4j public class ProtobufFileSerde implements BuiltInSerde { public static String name() { return "ProtobufFile"; } private static final ProtobufSchemaConverter SCHEMA_CONVERTER = new ProtobufSchemaConverter(); private Map messageDescriptorMap = new HashMap<>(); private Map keyMessageDescriptorMap = new HashMap<>(); private Map descriptorPaths = new HashMap<>(); @Nullable private Descriptor defaultMessageDescriptor; @Nullable private Descriptor defaultKeyMessageDescriptor; @Override public boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { return Configuration.canBeAutoConfigured(kafkaClusterProperties); } @Override public void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { configure(Configuration.create(kafkaClusterProperties)); } @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { configure(Configuration.create(serdeProperties)); } @VisibleForTesting void configure(Configuration configuration) { if (configuration.defaultMessageDescriptor() == null && configuration.defaultKeyMessageDescriptor() == null && configuration.messageDescriptorMap().isEmpty() && configuration.keyMessageDescriptorMap().isEmpty()) { throw new ValidationException("Neither default, not per-topic descriptors defined for " + name() + " serde"); } this.defaultMessageDescriptor = configuration.defaultMessageDescriptor(); this.defaultKeyMessageDescriptor = configuration.defaultKeyMessageDescriptor(); this.descriptorPaths = configuration.descriptorPaths(); this.messageDescriptorMap = configuration.messageDescriptorMap(); this.keyMessageDescriptorMap = configuration.keyMessageDescriptorMap(); } @Override public Optional getDescription() { return Optional.empty(); } private Optional descriptorFor(String topic, Target type) { return type == Target.KEY ? Optional.ofNullable(keyMessageDescriptorMap.get(topic)) .or(() -> Optional.ofNullable(defaultKeyMessageDescriptor)) : Optional.ofNullable(messageDescriptorMap.get(topic)) .or(() -> Optional.ofNullable(defaultMessageDescriptor)); } @Override public boolean canDeserialize(String topic, Target type) { return descriptorFor(topic, type).isPresent(); } @Override public boolean canSerialize(String topic, Target type) { return descriptorFor(topic, type).isPresent(); } @Override public Serializer serializer(String topic, Target type) { var descriptor = descriptorFor(topic, type).orElseThrow(); return new Serializer() { @SneakyThrows @Override public byte[] serialize(String input) { DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor); JsonFormat.parser().merge(input, builder); return builder.build().toByteArray(); } }; } @Override public Deserializer deserializer(String topic, Target type) { var descriptor = descriptorFor(topic, type).orElseThrow(); return new Deserializer() { @SneakyThrows @Override public DeserializeResult deserialize(RecordHeaders headers, byte[] data) { var protoMsg = DynamicMessage.parseFrom(descriptor, new ByteArrayInputStream(data)); byte[] jsonFromProto = ProtobufSchemaUtils.toJson(protoMsg); var result = new String(jsonFromProto); return new DeserializeResult( result, DeserializeResult.Type.JSON, Map.of() ); } }; } @Override public Optional getSchema(String topic, Target type) { return descriptorFor(topic, type).map(this::toSchemaDescription); } private SchemaDescription toSchemaDescription(Descriptor descriptor) { Path path = descriptorPaths.get(descriptor); return new SchemaDescription( SCHEMA_CONVERTER.convert(path.toUri(), descriptor).toJson(), Map.of("messageName", descriptor.getFullName()) ); } @SneakyThrows private static String readFileAsString(Path path) { return Files.readString(path); } //---------------------------------------------------------------------------------------------------------------- @VisibleForTesting record Configuration(@Nullable Descriptor defaultMessageDescriptor, @Nullable Descriptor defaultKeyMessageDescriptor, Map descriptorPaths, Map messageDescriptorMap, Map keyMessageDescriptorMap) { static boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties) { Optional> protobufFiles = kafkaClusterProperties.getListProperty("protobufFiles", String.class); Optional protobufFilesDir = kafkaClusterProperties.getProperty("protobufFilesDir", String.class); return protobufFilesDir.isPresent() || protobufFiles.filter(files -> !files.isEmpty()).isPresent(); } static Configuration create(PropertyResolver properties) { var protobufSchemas = loadSchemas( properties.getListProperty("protobufFiles", String.class), properties.getProperty("protobufFilesDir", String.class) ); // Load all referenced message schemas and store their source proto file with the descriptors Map descriptorPaths = new HashMap<>(); Optional protobufMessageName = properties.getProperty("protobufMessageName", String.class); protobufMessageName.ifPresent(messageName -> addProtobufSchema(descriptorPaths, protobufSchemas, messageName)); Optional protobufMessageNameForKey = properties.getProperty("protobufMessageNameForKey", String.class); protobufMessageNameForKey .ifPresent(messageName -> addProtobufSchema(descriptorPaths, protobufSchemas, messageName)); Optional> protobufMessageNameByTopic = properties.getMapProperty("protobufMessageNameByTopic", String.class, String.class); protobufMessageNameByTopic .ifPresent(messageNamesByTopic -> addProtobufSchemas(descriptorPaths, protobufSchemas, messageNamesByTopic)); Optional> protobufMessageNameForKeyByTopic = properties.getMapProperty("protobufMessageNameForKeyByTopic", String.class, String.class); protobufMessageNameForKeyByTopic .ifPresent(messageNamesByTopic -> addProtobufSchemas(descriptorPaths, protobufSchemas, messageNamesByTopic)); // Fill dictionary for descriptor lookup by full message name Map descriptorMap = descriptorPaths.keySet().stream() .collect(Collectors.toMap(Descriptor::getFullName, Function.identity())); return new Configuration( protobufMessageName.map(descriptorMap::get).orElse(null), protobufMessageNameForKey.map(descriptorMap::get).orElse(null), descriptorPaths, protobufMessageNameByTopic.map(map -> populateDescriptors(descriptorMap, map)).orElse(Map.of()), protobufMessageNameForKeyByTopic.map(map -> populateDescriptors(descriptorMap, map)).orElse(Map.of()) ); } private static Map.Entry getDescriptorAndPath(Map protobufSchemas, String msgName) { return protobufSchemas.entrySet().stream() .filter(schema -> schema.getValue().toDescriptor(msgName) != null) .map(schema -> Map.entry(schema.getValue().toDescriptor(msgName), schema.getKey())) .findFirst() .orElseThrow(() -> new NullPointerException( "The given message type not found in protobuf definition: " + msgName)); } private static Map populateDescriptors(Map descriptorMap, Map messageNameMap) { Map descriptors = new HashMap<>(); for (Map.Entry entry : messageNameMap.entrySet()) { descriptors.put(entry.getKey(), descriptorMap.get(entry.getValue())); } return descriptors; } @VisibleForTesting static Map loadSchemas(Optional> protobufFiles, Optional protobufFilesDir) { if (protobufFilesDir.isPresent()) { if (protobufFiles.isPresent()) { log.warn("protobufFiles properties will be ignored, since protobufFilesDir provided"); } List loadedFiles = new ProtoSchemaLoader(protobufFilesDir.get()).load(); Map allPaths = loadedFiles.stream() .collect(Collectors.toMap(f -> f.getLocation().getPath(), ProtoFile::toElement)); return loadedFiles.stream() .collect(Collectors.toMap( f -> Path.of(f.getLocation().getBase(), f.getLocation().getPath()), f -> new ProtobufSchema(f.toElement(), List.of(), allPaths))); } //Supporting for backward-compatibility. Normally, protobufFilesDir setting should be used return protobufFiles.stream() .flatMap(Collection::stream) .distinct() .map(Path::of) .collect(Collectors.toMap(path -> path, path -> new ProtobufSchema(readFileAsString(path)))); } private static void addProtobufSchema(Map descriptorPaths, Map protobufSchemas, String messageName) { var descriptorAndPath = getDescriptorAndPath(protobufSchemas, messageName); descriptorPaths.put(descriptorAndPath.getKey(), descriptorAndPath.getValue()); } private static void addProtobufSchemas(Map descriptorPaths, Map protobufSchemas, Map messageNamesByTopic) { messageNamesByTopic.values().stream() .map(msgName -> getDescriptorAndPath(protobufSchemas, msgName)) .forEach(entry -> descriptorPaths.put(entry.getKey(), entry.getValue())); } } static class ProtoSchemaLoader { private final Path baseLocation; ProtoSchemaLoader(String baseLocationStr) { this.baseLocation = Path.of(baseLocationStr); if (!Files.isReadable(baseLocation)) { throw new ValidationException("proto files directory not readable"); } } List load() { Map knownTypes = knownProtoFiles(); Map filesByLocations = new HashMap<>(); filesByLocations.putAll(knownTypes); filesByLocations.putAll(loadFilesWithLocations()); Linker linker = new Linker( createFilesLoader(filesByLocations), new ErrorCollector(), true, true ); var schema = linker.link(filesByLocations.values()); linker.getErrors().throwIfNonEmpty(); return schema.getProtoFiles() .stream() .filter(p -> !knownTypes.containsKey(p.getLocation().getPath())) //filtering known types .toList(); } private Map knownProtoFiles() { return Stream.of( loadKnownProtoFile("google/type/color.proto", ColorProto.getDescriptor()), loadKnownProtoFile("google/type/date.proto", DateProto.getDescriptor()), loadKnownProtoFile("google/type/datetime.proto", DateTimeProto.getDescriptor()), loadKnownProtoFile("google/type/dayofweek.proto", DayOfWeekProto.getDescriptor()), loadKnownProtoFile("google/type/decimal.proto", com.google.type.DecimalProto.getDescriptor()), loadKnownProtoFile("google/type/expr.proto", ExprProto.getDescriptor()), loadKnownProtoFile("google/type/fraction.proto", FractionProto.getDescriptor()), loadKnownProtoFile("google/type/interval.proto", IntervalProto.getDescriptor()), loadKnownProtoFile("google/type/latlng.proto", LatLngProto.getDescriptor()), loadKnownProtoFile("google/type/money.proto", MoneyProto.getDescriptor()), loadKnownProtoFile("google/type/month.proto", MonthProto.getDescriptor()), loadKnownProtoFile("google/type/phone_number.proto", PhoneNumberProto.getDescriptor()), loadKnownProtoFile("google/type/postal_address.proto", PostalAddressProto.getDescriptor()), loadKnownProtoFile("google/type/quaternion.prot", QuaternionProto.getDescriptor()), loadKnownProtoFile("google/type/timeofday.proto", TimeOfDayProto.getDescriptor()), loadKnownProtoFile("google/protobuf/any.proto", AnyProto.getDescriptor()), loadKnownProtoFile("google/protobuf/api.proto", ApiProto.getDescriptor()), loadKnownProtoFile("google/protobuf/descriptor.proto", DescriptorProtos.getDescriptor()), loadKnownProtoFile("google/protobuf/duration.proto", DurationProto.getDescriptor()), loadKnownProtoFile("google/protobuf/empty.proto", EmptyProto.getDescriptor()), loadKnownProtoFile("google/protobuf/field_mask.proto", FieldMaskProto.getDescriptor()), loadKnownProtoFile("google/protobuf/source_context.proto", SourceContextProto.getDescriptor()), loadKnownProtoFile("google/protobuf/struct.proto", StructProto.getDescriptor()), loadKnownProtoFile("google/protobuf/timestamp.proto", TimestampProto.getDescriptor()), loadKnownProtoFile("google/protobuf/type.proto", TypeProto.getDescriptor()), loadKnownProtoFile("google/protobuf/wrappers.proto", WrappersProto.getDescriptor()) ).collect(Collectors.toMap(p -> p.getLocation().getPath(), p -> p)); } private ProtoFile loadKnownProtoFile(String path, Descriptors.FileDescriptor fileDescriptor) { String protoFileString = null; // know type file contains either message or enum if (!fileDescriptor.getMessageTypes().isEmpty()) { protoFileString = new ProtobufSchema(fileDescriptor.getMessageTypes().get(0)).canonicalString(); } else if (!fileDescriptor.getEnumTypes().isEmpty()) { protoFileString = new ProtobufSchema(fileDescriptor.getEnumTypes().get(0)).canonicalString(); } else { throw new IllegalStateException(); } return ProtoFile.Companion.get(ProtoParser.Companion.parse(Location.get(path), protoFileString)); } private Loader createFilesLoader(Map files) { return new Loader() { @Override public @NotNull ProtoFile load(@NotNull String path) { return Preconditions.checkNotNull(files.get(path), "ProtoFile not found for import '%s'", path); } @Override public @NotNull Loader withErrors(@NotNull ErrorCollector errorCollector) { return this; } }; } @SneakyThrows private Map loadFilesWithLocations() { Map filesByLocations = new HashMap<>(); try (var files = Files.walk(baseLocation)) { files.filter(p -> !Files.isDirectory(p) && p.toString().endsWith(".proto")) .forEach(path -> { // relative path will be used as "import" statement String relativePath = baseLocation.relativize(path).toString(); var protoFileElement = ProtoParser.Companion.parse( Location.get(baseLocation.toString(), relativePath), readFileAsString(path) ); filesByLocations.put(relativePath, ProtoFile.Companion.get(protoFileElement)); }); } return filesByLocations; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/ProtobufRawSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.protobuf.UnknownFieldSet; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.RecordHeaders; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; public class ProtobufRawSerde implements BuiltInSerde { public static String name() { return "ProtobufDecodeRaw"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canSerialize(String topic, Target type) { return false; } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { throw new UnsupportedOperationException(); } @Override public Deserializer deserializer(String topic, Target type) { return new Deserializer() { @SneakyThrows @Override public DeserializeResult deserialize(RecordHeaders headers, byte[] data) { try { UnknownFieldSet unknownFields = UnknownFieldSet.parseFrom(data); return new DeserializeResult(unknownFields.toString(), DeserializeResult.Type.STRING, Map.of()); } catch (Exception e) { throw new ValidationException(e.getMessage()); } } }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/StringSerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Optional; public class StringSerde implements BuiltInSerde { public static String name() { return "String"; } private Charset encoding = StandardCharsets.UTF_8; @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { serdeProperties.getProperty("encoding", String.class) .map(Charset::forName) .ifPresent(e -> StringSerde.this.encoding = e); } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> input.getBytes(encoding); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( new String(data, encoding), DeserializeResult.Type.STRING, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/UInt32Serde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.common.primitives.Ints; import com.google.common.primitives.UnsignedInteger; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Map; import java.util.Optional; public class UInt32Serde implements BuiltInSerde { public static String name() { return "UInt32"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.of( new SchemaDescription( String.format( "{ " + " \"type\" : \"integer\", " + " \"minimum\" : 0, " + " \"maximum\" : %s" + "}", UnsignedInteger.MAX_VALUE ), Map.of() ) ); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> Ints.toByteArray(Integer.parseUnsignedInt(input)); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( UnsignedInteger.fromIntBits(Ints.fromByteArray(data)).toString(), DeserializeResult.Type.JSON, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/UInt64Serde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedLong; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.util.Map; import java.util.Optional; public class UInt64Serde implements BuiltInSerde { public static String name() { return "UInt64"; } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.of( new SchemaDescription( String.format( "{ " + " \"type\" : \"integer\", " + " \"minimum\" : 0, " + " \"maximum\" : %s " + "}", UnsignedLong.MAX_VALUE ), Map.of() ) ); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> Longs.toByteArray(Long.parseUnsignedLong(input)); } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> new DeserializeResult( UnsignedLong.fromLongBits(Longs.fromByteArray(data)).toString(), DeserializeResult.Type.JSON, Map.of() ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/UuidBinarySerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.RecordHeaders; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; import java.util.UUID; public class UuidBinarySerde implements BuiltInSerde { public static String name() { return "UUIDBinary"; } private boolean mostSignificantBitsFirst = true; @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { serdeProperties.getProperty("mostSignificantBitsFirst", Boolean.class) .ifPresent(msb -> UuidBinarySerde.this.mostSignificantBitsFirst = msb); } @Override public Optional getDescription() { return Optional.empty(); } @Override public Optional getSchema(String topic, Target type) { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { return true; } @Override public boolean canSerialize(String topic, Target type) { return true; } @Override public Serializer serializer(String topic, Target type) { return input -> { UUID uuid = UUID.fromString(input); ByteBuffer bb = ByteBuffer.wrap(new byte[16]); if (mostSignificantBitsFirst) { bb.putLong(uuid.getMostSignificantBits()); bb.putLong(uuid.getLeastSignificantBits()); } else { bb.putLong(uuid.getLeastSignificantBits()); bb.putLong(uuid.getMostSignificantBits()); } return bb.array(); }; } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> { if (data.length != 16) { throw new ValidationException("UUID data should be 16 bytes, but it is " + data.length); } ByteBuffer bb = ByteBuffer.wrap(data); long msb = bb.getLong(); long lsb = bb.getLong(); UUID uuid = mostSignificantBitsFirst ? new UUID(msb, lsb) : new UUID(lsb, msb); return new DeserializeResult( uuid.toString(), DeserializeResult.Type.STRING, Map.of() ); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/sr/MessageFormatter.java ================================================ package com.provectus.kafka.ui.serdes.builtin.sr; import com.fasterxml.jackson.databind.JsonNode; import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; import com.provectus.kafka.ui.util.jsonschema.JsonAvroConversion; import io.confluent.kafka.schemaregistry.avro.AvroSchemaUtils; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import io.confluent.kafka.serializers.KafkaAvroDeserializer; import io.confluent.kafka.serializers.KafkaAvroDeserializerConfig; import io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer; import io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer; import java.util.Map; import lombok.SneakyThrows; interface MessageFormatter { String format(String topic, byte[] value); static Map createMap(SchemaRegistryClient schemaRegistryClient) { return Map.of( SchemaType.AVRO, new AvroMessageFormatter(schemaRegistryClient), SchemaType.JSON, new JsonSchemaMessageFormatter(schemaRegistryClient), SchemaType.PROTOBUF, new ProtobufMessageFormatter(schemaRegistryClient) ); } class AvroMessageFormatter implements MessageFormatter { private final KafkaAvroDeserializer avroDeserializer; AvroMessageFormatter(SchemaRegistryClient client) { this.avroDeserializer = new KafkaAvroDeserializer(client); this.avroDeserializer.configure( Map.of( AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "wontbeused", KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG, false, KafkaAvroDeserializerConfig.SCHEMA_REFLECTION_CONFIG, false, KafkaAvroDeserializerConfig.AVRO_USE_LOGICAL_TYPE_CONVERTERS_CONFIG, true ), false ); } @Override public String format(String topic, byte[] value) { Object deserialized = avroDeserializer.deserialize(topic, value); var schema = AvroSchemaUtils.getSchema(deserialized); return JsonAvroConversion.convertAvroToJson(deserialized, schema).toString(); } } class ProtobufMessageFormatter implements MessageFormatter { private final KafkaProtobufDeserializer protobufDeserializer; ProtobufMessageFormatter(SchemaRegistryClient client) { this.protobufDeserializer = new KafkaProtobufDeserializer<>(client); } @Override @SneakyThrows public String format(String topic, byte[] value) { final Message message = protobufDeserializer.deserialize(topic, value); return JsonFormat.printer() .includingDefaultValueFields() .omittingInsignificantWhitespace() .preservingProtoFieldNames() .print(message); } } class JsonSchemaMessageFormatter implements MessageFormatter { private final KafkaJsonSchemaDeserializer jsonSchemaDeserializer; JsonSchemaMessageFormatter(SchemaRegistryClient client) { this.jsonSchemaDeserializer = new KafkaJsonSchemaDeserializer<>(client); } @Override public String format(String topic, byte[] value) { JsonNode json = jsonSchemaDeserializer.deserialize(topic, value); return json.toString(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/sr/SchemaRegistrySerde.java ================================================ package com.provectus.kafka.ui.serdes.builtin.sr; import static com.provectus.kafka.ui.serdes.builtin.sr.Serialize.serializeAvro; import static com.provectus.kafka.ui.serdes.builtin.sr.Serialize.serializeJson; import static com.provectus.kafka.ui.serdes.builtin.sr.Serialize.serializeProto; import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.BASIC_AUTH_CREDENTIALS_SOURCE; import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.USER_INFO_CONFIG; import com.google.common.annotations.VisibleForTesting; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serdes.BuiltInSerde; import com.provectus.kafka.ui.util.jsonschema.AvroJsonSchemaConverter; import com.provectus.kafka.ui.util.jsonschema.ProtobufSchemaConverter; import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClientConfig; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.confluent.kafka.schemaregistry.json.JsonSchema; import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; import java.net.URI; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; import lombok.SneakyThrows; import org.apache.kafka.common.config.SslConfigs; public class SchemaRegistrySerde implements BuiltInSerde { private static final byte SR_PAYLOAD_MAGIC_BYTE = 0x0; private static final int SR_PAYLOAD_PREFIX_LENGTH = 5; public static String name() { return "SchemaRegistry"; } private static final String SCHEMA_REGISTRY = "schemaRegistry"; private SchemaRegistryClient schemaRegistryClient; private List schemaRegistryUrls; private String valueSchemaNameTemplate; private String keySchemaNameTemplate; private boolean checkSchemaExistenceForDeserialize; private Map schemaRegistryFormatters; @Override public boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { return kafkaClusterProperties.getListProperty(SCHEMA_REGISTRY, String.class) .filter(lst -> !lst.isEmpty()) .isPresent(); } @Override public void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { var urls = kafkaClusterProperties.getListProperty(SCHEMA_REGISTRY, String.class) .filter(lst -> !lst.isEmpty()) .orElseThrow(() -> new ValidationException("No urls provided for schema registry")); configure( urls, createSchemaRegistryClient( urls, kafkaClusterProperties.getProperty("schemaRegistryAuth.username", String.class).orElse(null), kafkaClusterProperties.getProperty("schemaRegistryAuth.password", String.class).orElse(null), kafkaClusterProperties.getProperty("schemaRegistrySsl.keystoreLocation", String.class).orElse(null), kafkaClusterProperties.getProperty("schemaRegistrySsl.keystorePassword", String.class).orElse(null), kafkaClusterProperties.getProperty("ssl.truststoreLocation", String.class).orElse(null), kafkaClusterProperties.getProperty("ssl.truststorePassword", String.class).orElse(null) ), kafkaClusterProperties.getProperty("schemaRegistryKeySchemaNameTemplate", String.class).orElse("%s-key"), kafkaClusterProperties.getProperty("schemaRegistrySchemaNameTemplate", String.class).orElse("%s-value"), kafkaClusterProperties.getProperty("schemaRegistryCheckSchemaExistenceForDeserialize", Boolean.class) .orElse(false) ); } @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { var urls = serdeProperties.getListProperty("url", String.class) .or(() -> kafkaClusterProperties.getListProperty(SCHEMA_REGISTRY, String.class)) .filter(lst -> !lst.isEmpty()) .orElseThrow(() -> new ValidationException("No urls provided for schema registry")); configure( urls, createSchemaRegistryClient( urls, serdeProperties.getProperty("username", String.class).orElse(null), serdeProperties.getProperty("password", String.class).orElse(null), serdeProperties.getProperty("keystoreLocation", String.class).orElse(null), serdeProperties.getProperty("keystorePassword", String.class).orElse(null), kafkaClusterProperties.getProperty("ssl.truststoreLocation", String.class).orElse(null), kafkaClusterProperties.getProperty("ssl.truststorePassword", String.class).orElse(null) ), serdeProperties.getProperty("keySchemaNameTemplate", String.class).orElse("%s-key"), serdeProperties.getProperty("schemaNameTemplate", String.class).orElse("%s-value"), serdeProperties.getProperty("checkSchemaExistenceForDeserialize", Boolean.class) .orElse(false) ); } @VisibleForTesting void configure( List schemaRegistryUrls, SchemaRegistryClient schemaRegistryClient, String keySchemaNameTemplate, String valueSchemaNameTemplate, boolean checkTopicSchemaExistenceForDeserialize) { this.schemaRegistryUrls = schemaRegistryUrls; this.schemaRegistryClient = schemaRegistryClient; this.keySchemaNameTemplate = keySchemaNameTemplate; this.valueSchemaNameTemplate = valueSchemaNameTemplate; this.schemaRegistryFormatters = MessageFormatter.createMap(schemaRegistryClient); this.checkSchemaExistenceForDeserialize = checkTopicSchemaExistenceForDeserialize; } private static SchemaRegistryClient createSchemaRegistryClient(List urls, @Nullable String username, @Nullable String password, @Nullable String keyStoreLocation, @Nullable String keyStorePassword, @Nullable String trustStoreLocation, @Nullable String trustStorePassword) { Map configs = new HashMap<>(); if (username != null && password != null) { configs.put(BASIC_AUTH_CREDENTIALS_SOURCE, "USER_INFO"); configs.put(USER_INFO_CONFIG, username + ":" + password); } else if (username != null) { throw new ValidationException( "You specified username but do not specified password"); } else if (password != null) { throw new ValidationException( "You specified password but do not specified username"); } // We require at least a truststore. The logic is done similar to SchemaRegistryService.securedWebClientOnTLS if (trustStoreLocation != null && trustStorePassword != null) { configs.put(SchemaRegistryClientConfig.CLIENT_NAMESPACE + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, trustStoreLocation); configs.put(SchemaRegistryClientConfig.CLIENT_NAMESPACE + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, trustStorePassword); } if (keyStoreLocation != null && keyStorePassword != null) { configs.put(SchemaRegistryClientConfig.CLIENT_NAMESPACE + SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keyStoreLocation); configs.put(SchemaRegistryClientConfig.CLIENT_NAMESPACE + SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, keyStorePassword); configs.put(SchemaRegistryClientConfig.CLIENT_NAMESPACE + SslConfigs.SSL_KEY_PASSWORD_CONFIG, keyStorePassword); } return new CachedSchemaRegistryClient( urls, 1_000, List.of(new AvroSchemaProvider(), new ProtobufSchemaProvider(), new JsonSchemaProvider()), configs ); } @Override public Optional getDescription() { return Optional.empty(); } @Override public boolean canDeserialize(String topic, Target type) { String subject = schemaSubject(topic, type); return !checkSchemaExistenceForDeserialize || getSchemaBySubject(subject).isPresent(); } @Override public boolean canSerialize(String topic, Target type) { String subject = schemaSubject(topic, type); return getSchemaBySubject(subject).isPresent(); } @Override public Optional getSchema(String topic, Target type) { String subject = schemaSubject(topic, type); return getSchemaBySubject(subject) .flatMap(schemaMetadata -> //schema can be not-found, when schema contexts configured improperly getSchemaById(schemaMetadata.getId()) .map(parsedSchema -> new SchemaDescription( convertSchema(schemaMetadata, parsedSchema), Map.of( "subject", subject, "schemaId", schemaMetadata.getId(), "latestVersion", schemaMetadata.getVersion(), "type", schemaMetadata.getSchemaType() // AVRO / PROTOBUF / JSON ) ))); } @SneakyThrows private String convertSchema(SchemaMetadata schema, ParsedSchema parsedSchema) { URI basePath = new URI(schemaRegistryUrls.get(0)) .resolve(Integer.toString(schema.getId())); SchemaType schemaType = SchemaType.fromString(schema.getSchemaType()) .orElseThrow(() -> new IllegalStateException("Unknown schema type: " + schema.getSchemaType())); return switch (schemaType) { case PROTOBUF -> new ProtobufSchemaConverter() .convert(basePath, ((ProtobufSchema) parsedSchema).toDescriptor()) .toJson(); case AVRO -> new AvroJsonSchemaConverter() .convert(basePath, ((AvroSchema) parsedSchema).rawSchema()) .toJson(); case JSON -> //need to use confluent JsonSchema since it includes resolved references ((JsonSchema) parsedSchema).rawSchema().toString(); }; } private Optional getSchemaById(int id) { return wrapWith404Handler(() -> schemaRegistryClient.getSchemaById(id)); } private Optional getSchemaBySubject(String subject) { return wrapWith404Handler(() -> schemaRegistryClient.getLatestSchemaMetadata(subject)); } @SneakyThrows private Optional wrapWith404Handler(Callable call) { try { return Optional.ofNullable(call.call()); } catch (RestClientException restClientException) { if (restClientException.getStatus() == 404) { return Optional.empty(); } else { throw new RuntimeException("Error calling SchemaRegistryClient", restClientException); } } } private String schemaSubject(String topic, Target type) { return String.format(type == Target.KEY ? keySchemaNameTemplate : valueSchemaNameTemplate, topic); } @Override public Serializer serializer(String topic, Target type) { String subject = schemaSubject(topic, type); SchemaMetadata meta = getSchemaBySubject(subject) .orElseThrow(() -> new ValidationException( String.format("No schema for subject '%s' found", subject))); ParsedSchema schema = getSchemaById(meta.getId()) .orElseThrow(() -> new IllegalStateException( String.format("Schema found for id %s, subject '%s'", meta.getId(), subject))); SchemaType schemaType = SchemaType.fromString(meta.getSchemaType()) .orElseThrow(() -> new IllegalStateException("Unknown schema type: " + meta.getSchemaType())); return switch (schemaType) { case PROTOBUF -> input -> serializeProto(schemaRegistryClient, topic, type, (ProtobufSchema) schema, meta.getId(), input); case AVRO -> input -> serializeAvro((AvroSchema) schema, meta.getId(), input); case JSON -> input -> serializeJson((JsonSchema) schema, meta.getId(), input); }; } @Override public Deserializer deserializer(String topic, Target type) { return (headers, data) -> { var schemaId = extractSchemaIdFromMsg(data); SchemaType format = getMessageFormatBySchemaId(schemaId); MessageFormatter formatter = schemaRegistryFormatters.get(format); return new DeserializeResult( formatter.format(topic, data), DeserializeResult.Type.JSON, Map.of( "schemaId", schemaId, "type", format.name() ) ); }; } private SchemaType getMessageFormatBySchemaId(int schemaId) { return getSchemaById(schemaId) .map(ParsedSchema::schemaType) .flatMap(SchemaType::fromString) .orElseThrow(() -> new ValidationException(String.format("Schema for id '%d' not found ", schemaId))); } private int extractSchemaIdFromMsg(byte[] data) { ByteBuffer buffer = ByteBuffer.wrap(data); if (buffer.remaining() >= SR_PAYLOAD_PREFIX_LENGTH && buffer.get() == SR_PAYLOAD_MAGIC_BYTE) { return buffer.getInt(); } throw new ValidationException( String.format( "Data doesn't contain magic byte and schema id prefix, so it can't be deserialized with %s serde", name()) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/sr/SchemaType.java ================================================ package com.provectus.kafka.ui.serdes.builtin.sr; import java.util.Optional; import org.apache.commons.lang3.EnumUtils; enum SchemaType { AVRO, JSON, PROTOBUF; public static Optional fromString(String typeString) { return Optional.ofNullable(EnumUtils.getEnum(SchemaType.class, typeString)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/serdes/builtin/sr/Serialize.java ================================================ package com.provectus.kafka.ui.serdes.builtin.sr; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; import com.google.protobuf.util.JsonFormat; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.util.annotation.KafkaClientInternalsDependant; import com.provectus.kafka.ui.util.jsonschema.JsonAvroConversion; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.avro.AvroSchemaUtils; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.json.JsonSchema; import io.confluent.kafka.schemaregistry.json.jackson.Jackson; import io.confluent.kafka.schemaregistry.protobuf.MessageIndexes; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import io.confluent.kafka.serializers.protobuf.AbstractKafkaProtobufSerializer; import io.confluent.kafka.serializers.subject.DefaultReferenceSubjectNameStrategy; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.HashMap; import lombok.SneakyThrows; import org.apache.avro.Schema; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DatumWriter; import org.apache.avro.io.EncoderFactory; final class Serialize { private static final byte MAGIC = 0x0; private static final ObjectMapper JSON_SERIALIZE_MAPPER = Jackson.newObjectMapper(); //from confluent package private Serialize() { } @KafkaClientInternalsDependant("AbstractKafkaJsonSchemaSerializer::serializeImpl") @SneakyThrows static byte[] serializeJson(JsonSchema schema, int schemaId, String value) { JsonNode json; try { json = JSON_SERIALIZE_MAPPER.readTree(value); } catch (JsonProcessingException e) { throw new ValidationException(String.format("'%s' is not valid json", value)); } try { schema.validate(json); } catch (org.everit.json.schema.ValidationException e) { throw new ValidationException( String.format("'%s' does not fit schema: %s", value, e.getAllMessages())); } try (var out = new ByteArrayOutputStream()) { out.write(MAGIC); out.write(schemaId(schemaId)); out.write(JSON_SERIALIZE_MAPPER.writeValueAsBytes(json)); return out.toByteArray(); } } @KafkaClientInternalsDependant("AbstractKafkaProtobufSerializer::serializeImpl") @SneakyThrows static byte[] serializeProto(SchemaRegistryClient srClient, String topic, Serde.Target target, ProtobufSchema schema, int schemaId, String input) { // flags are tuned like in ProtobufSerializer by default boolean normalizeSchema = false; boolean autoRegisterSchema = false; boolean useLatestVersion = true; boolean latestCompatStrict = true; boolean skipKnownTypes = true; schema = AbstractKafkaProtobufSerializer.resolveDependencies( srClient, normalizeSchema, autoRegisterSchema, useLatestVersion, latestCompatStrict, new HashMap<>(), skipKnownTypes, new DefaultReferenceSubjectNameStrategy(), topic, target == Serde.Target.KEY, schema ); DynamicMessage.Builder builder = schema.newMessageBuilder(); JsonFormat.parser().merge(input, builder); Message message = builder.build(); MessageIndexes indexes = schema.toMessageIndexes(message.getDescriptorForType().getFullName(), normalizeSchema); try (var out = new ByteArrayOutputStream()) { out.write(MAGIC); out.write(schemaId(schemaId)); out.write(indexes.toByteArray()); message.writeTo(out); return out.toByteArray(); } } @KafkaClientInternalsDependant("AbstractKafkaAvroSerializer::serializeImpl") @SneakyThrows static byte[] serializeAvro(AvroSchema schema, int schemaId, String input) { var avroObject = JsonAvroConversion.convertJsonToAvro(input, schema.rawSchema()); try (var out = new ByteArrayOutputStream()) { out.write(MAGIC); out.write(schemaId(schemaId)); Schema rawSchema = schema.rawSchema(); if (rawSchema.getType().equals(Schema.Type.BYTES)) { Preconditions.checkState( avroObject instanceof ByteBuffer, "Unrecognized bytes object of type: " + avroObject.getClass().getName() ); out.write(((ByteBuffer) avroObject).array()); } else { boolean useLogicalTypeConverters = true; BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); DatumWriter writer = (DatumWriter) AvroSchemaUtils.getDatumWriter(avroObject, rawSchema, useLogicalTypeConverters); writer.write(avroObject, encoder); encoder.flush(); } return out.toByteArray(); } } private static byte[] schemaId(int id) { return ByteBuffer.allocate(Integer.BYTES).putInt(id).array(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/AdminClientService.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.model.KafkaCluster; import reactor.core.publisher.Mono; public interface AdminClientService { Mono get(KafkaCluster cluster); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/AdminClientServiceImpl.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.util.SslPropertiesUtil; import java.io.Closeable; import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @Service @Slf4j public class AdminClientServiceImpl implements AdminClientService, Closeable { private static final int DEFAULT_CLIENT_TIMEOUT_MS = 30_000; private static final AtomicLong CLIENT_ID_SEQ = new AtomicLong(); private final Map adminClientCache = new ConcurrentHashMap<>(); private final int clientTimeout; public AdminClientServiceImpl(ClustersProperties clustersProperties) { this.clientTimeout = Optional.ofNullable(clustersProperties.getAdminClientTimeout()) .orElse(DEFAULT_CLIENT_TIMEOUT_MS); } @Override public Mono get(KafkaCluster cluster) { return Mono.justOrEmpty(adminClientCache.get(cluster.getName())) .switchIfEmpty(createAdminClient(cluster)) .map(e -> adminClientCache.computeIfAbsent(cluster.getName(), key -> e)); } private Mono createAdminClient(KafkaCluster cluster) { return Mono.fromSupplier(() -> { Properties properties = new Properties(); SslPropertiesUtil.addKafkaSslProperties(cluster.getOriginalProperties().getSsl(), properties); properties.putAll(cluster.getProperties()); properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); properties.putIfAbsent(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, clientTimeout); properties.putIfAbsent( AdminClientConfig.CLIENT_ID_CONFIG, "kafka-ui-admin-" + Instant.now().getEpochSecond() + "-" + CLIENT_ID_SEQ.incrementAndGet() ); return AdminClient.create(properties); }).flatMap(ac -> ReactiveAdminClient.create(ac).doOnError(th -> ac.close())) .onErrorMap(th -> new IllegalStateException( "Error while creating AdminClient for Cluster " + cluster.getName(), th)); } @Override public void close() { adminClientCache.values().forEach(ReactiveAdminClient::close); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ApplicationInfoService.java ================================================ package com.provectus.kafka.ui.service; import static com.provectus.kafka.ui.model.ApplicationInfoDTO.EnabledFeaturesEnum; import com.provectus.kafka.ui.model.ApplicationInfoBuildDTO; import com.provectus.kafka.ui.model.ApplicationInfoDTO; import com.provectus.kafka.ui.model.ApplicationInfoLatestReleaseDTO; import com.provectus.kafka.ui.util.DynamicConfigOperations; import com.provectus.kafka.ui.util.GithubReleaseInfo; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Properties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.info.BuildProperties; import org.springframework.boot.info.GitProperties; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Service public class ApplicationInfoService { private final GithubReleaseInfo githubReleaseInfo = new GithubReleaseInfo(); private final DynamicConfigOperations dynamicConfigOperations; private final BuildProperties buildProperties; private final GitProperties gitProperties; public ApplicationInfoService(DynamicConfigOperations dynamicConfigOperations, @Autowired(required = false) BuildProperties buildProperties, @Autowired(required = false) GitProperties gitProperties) { this.dynamicConfigOperations = dynamicConfigOperations; this.buildProperties = Optional.ofNullable(buildProperties).orElse(new BuildProperties(new Properties())); this.gitProperties = Optional.ofNullable(gitProperties).orElse(new GitProperties(new Properties())); } public ApplicationInfoDTO getApplicationInfo() { var releaseInfo = githubReleaseInfo.get(); return new ApplicationInfoDTO() .build(getBuildInfo(releaseInfo)) .enabledFeatures(getEnabledFeatures()) .latestRelease(convert(releaseInfo)); } private ApplicationInfoLatestReleaseDTO convert(GithubReleaseInfo.GithubReleaseDto releaseInfo) { return new ApplicationInfoLatestReleaseDTO() .htmlUrl(releaseInfo.html_url()) .publishedAt(releaseInfo.published_at()) .versionTag(releaseInfo.tag_name()); } private ApplicationInfoBuildDTO getBuildInfo(GithubReleaseInfo.GithubReleaseDto release) { return new ApplicationInfoBuildDTO() .isLatestRelease(release.tag_name() != null && release.tag_name().equals(buildProperties.getVersion())) .commitId(gitProperties.getShortCommitId()) .version(buildProperties.getVersion()) .buildTime(buildProperties.getTime() != null ? DateTimeFormatter.ISO_INSTANT.format(buildProperties.getTime()) : null); } private List getEnabledFeatures() { var enabledFeatures = new ArrayList(); if (dynamicConfigOperations.dynamicConfigEnabled()) { enabledFeatures.add(EnabledFeaturesEnum.DYNAMIC_CONFIG); } return enabledFeatures; } // updating on startup and every hour @Scheduled(fixedRateString = "${github-release-info-update-rate:3600000}") public void updateGithubReleaseInfo() { githubReleaseInfo.refresh().block(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/BrokerService.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.exception.InvalidRequestApiException; import com.provectus.kafka.ui.exception.LogDirNotFoundApiException; import com.provectus.kafka.ui.exception.NotFoundException; import com.provectus.kafka.ui.exception.TopicOrPartitionNotFoundException; import com.provectus.kafka.ui.mapper.DescribeLogDirsMapper; import com.provectus.kafka.ui.model.BrokerLogdirUpdateDTO; import com.provectus.kafka.ui.model.BrokersLogdirsDTO; import com.provectus.kafka.ui.model.InternalBroker; import com.provectus.kafka.ui.model.InternalBrokerConfig; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.PartitionDistributionStats; import com.provectus.kafka.ui.service.metrics.RawMetric; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.LogDirNotFoundException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Service @RequiredArgsConstructor @Slf4j public class BrokerService { private final StatisticsCache statisticsCache; private final AdminClientService adminClientService; private final DescribeLogDirsMapper describeLogDirsMapper; private Mono>> loadBrokersConfig( KafkaCluster cluster, List brokersIds) { return adminClientService.get(cluster).flatMap(ac -> ac.loadBrokersConfig(brokersIds)); } private Mono> loadBrokersConfig( KafkaCluster cluster, Integer brokerId) { return loadBrokersConfig(cluster, Collections.singletonList(brokerId)) .map(map -> map.values().stream().findFirst().orElse(List.of())); } private Flux getBrokersConfig(KafkaCluster cluster, Integer brokerId) { if (statisticsCache.get(cluster).getClusterDescription().getNodes() .stream().noneMatch(node -> node.id() == brokerId)) { return Flux.error( new NotFoundException(String.format("Broker with id %s not found", brokerId))); } return loadBrokersConfig(cluster, brokerId) .map(list -> list.stream() .map(InternalBrokerConfig::from) .collect(Collectors.toList())) .flatMapMany(Flux::fromIterable); } public Flux getBrokers(KafkaCluster cluster) { var stats = statisticsCache.get(cluster); var partitionsDistribution = PartitionDistributionStats.create(stats); return adminClientService .get(cluster) .flatMap(ReactiveAdminClient::describeCluster) .map(description -> description.getNodes().stream() .map(node -> new InternalBroker(node, partitionsDistribution, stats)) .collect(Collectors.toList())) .flatMapMany(Flux::fromIterable); } public Mono updateBrokerLogDir(KafkaCluster cluster, Integer broker, BrokerLogdirUpdateDTO brokerLogDir) { return adminClientService.get(cluster) .flatMap(ac -> updateBrokerLogDir(ac, brokerLogDir, broker)); } private Mono updateBrokerLogDir(ReactiveAdminClient admin, BrokerLogdirUpdateDTO b, Integer broker) { Map req = Map.of( new TopicPartitionReplica(b.getTopic(), b.getPartition(), broker), b.getLogDir()); return admin.alterReplicaLogDirs(req) .onErrorResume(UnknownTopicOrPartitionException.class, e -> Mono.error(new TopicOrPartitionNotFoundException())) .onErrorResume(LogDirNotFoundException.class, e -> Mono.error(new LogDirNotFoundApiException())) .doOnError(e -> log.error("Unexpected error", e)); } public Mono updateBrokerConfigByName(KafkaCluster cluster, Integer broker, String name, String value) { return adminClientService.get(cluster) .flatMap(ac -> ac.updateBrokerConfigByName(broker, name, value)) .onErrorResume(InvalidRequestException.class, e -> Mono.error(new InvalidRequestApiException(e.getMessage()))) .doOnError(e -> log.error("Unexpected error", e)); } private Mono>> getClusterLogDirs( KafkaCluster cluster, List reqBrokers) { return adminClientService.get(cluster) .flatMap(admin -> { List brokers = statisticsCache.get(cluster).getClusterDescription().getNodes() .stream() .map(Node::id) .collect(Collectors.toList()); if (!reqBrokers.isEmpty()) { brokers.retainAll(reqBrokers); } return admin.describeLogDirs(brokers); }) .onErrorResume(TimeoutException.class, (TimeoutException e) -> { log.error("Error during fetching log dirs", e); return Mono.just(new HashMap<>()); }); } public Flux getAllBrokersLogdirs(KafkaCluster cluster, List brokers) { return getClusterLogDirs(cluster, brokers) .map(describeLogDirsMapper::toBrokerLogDirsList) .flatMapMany(Flux::fromIterable); } public Flux getBrokerConfig(KafkaCluster cluster, Integer brokerId) { return getBrokersConfig(cluster, brokerId); } public Mono> getBrokerMetrics(KafkaCluster cluster, Integer brokerId) { return Mono.justOrEmpty(statisticsCache.get(cluster).getMetrics().getPerBrokerMetrics().get(brokerId)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClusterService.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.model.ClusterDTO; import com.provectus.kafka.ui.model.ClusterMetricsDTO; import com.provectus.kafka.ui.model.ClusterStatsDTO; import com.provectus.kafka.ui.model.InternalClusterState; import com.provectus.kafka.ui.model.KafkaCluster; import java.util.List; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @Service @RequiredArgsConstructor @Slf4j public class ClusterService { private final StatisticsCache statisticsCache; private final ClustersStorage clustersStorage; private final ClusterMapper clusterMapper; private final StatisticsService statisticsService; public List getClusters() { return clustersStorage.getKafkaClusters() .stream() .map(c -> clusterMapper.toCluster(new InternalClusterState(c, statisticsCache.get(c)))) .collect(Collectors.toList()); } public Mono getClusterStats(KafkaCluster cluster) { return Mono.justOrEmpty( clusterMapper.toClusterStats( new InternalClusterState(cluster, statisticsCache.get(cluster))) ); } public Mono getClusterMetrics(KafkaCluster cluster) { return Mono.just( clusterMapper.toClusterMetrics( statisticsCache.get(cluster).getMetrics())); } public Mono updateCluster(KafkaCluster cluster) { return statisticsService.updateCache(cluster) .map(metrics -> clusterMapper.toCluster(new InternalClusterState(cluster, metrics))); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStatisticsScheduler.java ================================================ package com.provectus.kafka.ui.service; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import reactor.core.publisher.Flux; import reactor.core.scheduler.Schedulers; @Component @RequiredArgsConstructor @Slf4j public class ClustersStatisticsScheduler { private final ClustersStorage clustersStorage; private final StatisticsService statisticsService; @Scheduled(fixedRateString = "${kafka.update-metrics-rate-millis:30000}") public void updateStatistics() { Flux.fromIterable(clustersStorage.getKafkaClusters()) .parallel() .runOn(Schedulers.parallel()) .flatMap(cluster -> { log.debug("Start getting metrics for kafkaCluster: {}", cluster.getName()); return statisticsService.updateCache(cluster) .doOnSuccess(m -> log.debug("Metrics updated for cluster: {}", cluster.getName())); }) .then() .block(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ClustersStorage.java ================================================ package com.provectus.kafka.ui.service; import com.google.common.collect.ImmutableMap; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.KafkaCluster; import java.util.Collection; import java.util.Optional; import org.springframework.stereotype.Component; @Component public class ClustersStorage { private final ImmutableMap kafkaClusters; public ClustersStorage(ClustersProperties properties, KafkaClusterFactory factory) { var builder = ImmutableMap.builder(); properties.getClusters().forEach(c -> builder.put(c.getName(), factory.create(properties, c))); this.kafkaClusters = builder.build(); } public Collection getKafkaClusters() { return kafkaClusters.values(); } public Optional getClusterByName(String clusterName) { return Optional.ofNullable(kafkaClusters.get(clusterName)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ConsumerGroupService.java ================================================ package com.provectus.kafka.ui.service; import com.google.common.collect.Streams; import com.google.common.collect.Table; import com.provectus.kafka.ui.emitter.EnhancedConsumer; import com.provectus.kafka.ui.model.ConsumerGroupOrderingDTO; import com.provectus.kafka.ui.model.InternalConsumerGroup; import com.provectus.kafka.ui.model.InternalTopicConsumerGroup; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.SortOrderDTO; import com.provectus.kafka.ui.service.rbac.AccessControlService; import com.provectus.kafka.ui.util.ApplicationMetrics; import com.provectus.kafka.ui.util.SslPropertiesUtil; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.function.ToIntFunction; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.clients.admin.ConsumerGroupListing; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.ConsumerGroupState; import org.apache.kafka.common.TopicPartition; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @Service @RequiredArgsConstructor public class ConsumerGroupService { private final AdminClientService adminClientService; private final AccessControlService accessControlService; private Mono> getConsumerGroups( ReactiveAdminClient ac, List descriptions) { var groupNames = descriptions.stream().map(ConsumerGroupDescription::groupId).toList(); // 1. getting committed offsets for all groups return ac.listConsumerGroupOffsets(groupNames, null) .flatMap((Table committedOffsets) -> { // 2. getting end offsets for partitions with committed offsets return ac.listOffsets(committedOffsets.columnKeySet(), OffsetSpec.latest(), false) .map(endOffsets -> descriptions.stream() .map(desc -> { var groupOffsets = committedOffsets.row(desc.groupId()); var endOffsetsForGroup = new HashMap<>(endOffsets); endOffsetsForGroup.keySet().retainAll(groupOffsets.keySet()); // 3. gathering description & offsets return InternalConsumerGroup.create(desc, groupOffsets, endOffsetsForGroup); }) .collect(Collectors.toList())); }); } public Mono> getConsumerGroupsForTopic(KafkaCluster cluster, String topic) { return adminClientService.get(cluster) // 1. getting topic's end offsets .flatMap(ac -> ac.listTopicOffsets(topic, OffsetSpec.latest(), false) .flatMap(endOffsets -> { var tps = new ArrayList<>(endOffsets.keySet()); // 2. getting all consumer groups return describeConsumerGroups(ac) .flatMap((List groups) -> { // 3. trying to find committed offsets for topic var groupNames = groups.stream().map(ConsumerGroupDescription::groupId).toList(); return ac.listConsumerGroupOffsets(groupNames, tps).map(offsets -> groups.stream() // 4. keeping only groups that relates to topic .filter(g -> isConsumerGroupRelatesToTopic(topic, g, offsets.containsRow(g.groupId()))) .map(g -> // 5. constructing results InternalTopicConsumerGroup.create(topic, g, offsets.row(g.groupId()), endOffsets)) .toList() ); } ); })); } private boolean isConsumerGroupRelatesToTopic(String topic, ConsumerGroupDescription description, boolean hasCommittedOffsets) { boolean hasActiveMembersForTopic = description.members() .stream() .anyMatch(m -> m.assignment().topicPartitions().stream().anyMatch(tp -> tp.topic().equals(topic))); return hasActiveMembersForTopic || hasCommittedOffsets; } public record ConsumerGroupsPage(List consumerGroups, int totalPages) { } private record GroupWithDescr(InternalConsumerGroup icg, ConsumerGroupDescription cgd) { } public Mono getConsumerGroupsPage( KafkaCluster cluster, int pageNum, int perPage, @Nullable String search, ConsumerGroupOrderingDTO orderBy, SortOrderDTO sortOrderDto) { return adminClientService.get(cluster).flatMap(ac -> ac.listConsumerGroups() .map(listing -> search == null ? listing : listing.stream() .filter(g -> StringUtils.containsIgnoreCase(g.groupId(), search)) .toList() ) .flatMapIterable(lst -> lst) .filterWhen(cg -> accessControlService.isConsumerGroupAccessible(cg.groupId(), cluster.getName())) .collectList() .flatMap(allGroups -> loadSortedDescriptions(ac, allGroups, pageNum, perPage, orderBy, sortOrderDto) .flatMap(descriptions -> getConsumerGroups(ac, descriptions) .map(page -> new ConsumerGroupsPage( page, (allGroups.size() / perPage) + (allGroups.size() % perPage == 0 ? 0 : 1)))))); } private Mono> loadSortedDescriptions(ReactiveAdminClient ac, List groups, int pageNum, int perPage, ConsumerGroupOrderingDTO orderBy, SortOrderDTO sortOrderDto) { return switch (orderBy) { case NAME -> { Comparator comparator = Comparator.comparing(ConsumerGroupListing::groupId); yield loadDescriptionsByListings(ac, groups, comparator, pageNum, perPage, sortOrderDto); } case STATE -> { ToIntFunction statesPriorities = cg -> switch (cg.state().orElse(ConsumerGroupState.UNKNOWN)) { case STABLE -> 0; case COMPLETING_REBALANCE -> 1; case PREPARING_REBALANCE -> 2; case EMPTY -> 3; case DEAD -> 4; case UNKNOWN -> 5; }; var comparator = Comparator.comparingInt(statesPriorities); yield loadDescriptionsByListings(ac, groups, comparator, pageNum, perPage, sortOrderDto); } case MEMBERS -> { var comparator = Comparator.comparingInt(cg -> cg.members().size()); var groupNames = groups.stream().map(ConsumerGroupListing::groupId).toList(); yield ac.describeConsumerGroups(groupNames) .map(descriptions -> sortAndPaginate(descriptions.values(), comparator, pageNum, perPage, sortOrderDto).toList()); } case MESSAGES_BEHIND -> { Comparator comparator = Comparator.comparingLong(gwd -> gwd.icg.getConsumerLag() == null ? 0L : gwd.icg.getConsumerLag()); yield loadDescriptionsByInternalConsumerGroups(ac, groups, comparator, pageNum, perPage, sortOrderDto); } case TOPIC_NUM -> { Comparator comparator = Comparator.comparingInt(gwd -> gwd.icg.getTopicNum()); yield loadDescriptionsByInternalConsumerGroups(ac, groups, comparator, pageNum, perPage, sortOrderDto); } }; } private Mono> loadDescriptionsByListings(ReactiveAdminClient ac, List listings, Comparator comparator, int pageNum, int perPage, SortOrderDTO sortOrderDto) { List sortedGroups = sortAndPaginate(listings, comparator, pageNum, perPage, sortOrderDto) .map(ConsumerGroupListing::groupId) .toList(); return ac.describeConsumerGroups(sortedGroups) .map(descrMap -> sortedGroups.stream().map(descrMap::get).toList()); } private Stream sortAndPaginate(Collection collection, Comparator comparator, int pageNum, int perPage, SortOrderDTO sortOrderDto) { return collection.stream() .sorted(sortOrderDto == SortOrderDTO.ASC ? comparator : comparator.reversed()) .skip((long) (pageNum - 1) * perPage) .limit(perPage); } private Mono> describeConsumerGroups(ReactiveAdminClient ac) { return ac.listConsumerGroupNames() .flatMap(ac::describeConsumerGroups) .map(cgs -> new ArrayList<>(cgs.values())); } private Mono> loadDescriptionsByInternalConsumerGroups(ReactiveAdminClient ac, List groups, Comparator comparator, int pageNum, int perPage, SortOrderDTO sortOrderDto) { var groupNames = groups.stream().map(ConsumerGroupListing::groupId).toList(); return ac.describeConsumerGroups(groupNames) .flatMap(descriptionsMap -> { List descriptions = descriptionsMap.values().stream().toList(); return getConsumerGroups(ac, descriptions) .map(icg -> Streams.zip(icg.stream(), descriptions.stream(), GroupWithDescr::new).toList()) .map(gwd -> sortAndPaginate(gwd, comparator, pageNum, perPage, sortOrderDto) .map(GroupWithDescr::cgd).toList()); } ); } public Mono getConsumerGroupDetail(KafkaCluster cluster, String consumerGroupId) { return adminClientService.get(cluster) .flatMap(ac -> ac.describeConsumerGroups(List.of(consumerGroupId)) .filter(m -> m.containsKey(consumerGroupId)) .map(r -> r.get(consumerGroupId)) .flatMap(descr -> getConsumerGroups(ac, List.of(descr)) .filter(groups -> !groups.isEmpty()) .map(groups -> groups.get(0)))); } public Mono deleteConsumerGroupById(KafkaCluster cluster, String groupId) { return adminClientService.get(cluster) .flatMap(adminClient -> adminClient.deleteConsumerGroups(List.of(groupId))); } public EnhancedConsumer createConsumer(KafkaCluster cluster) { return createConsumer(cluster, Map.of()); } public EnhancedConsumer createConsumer(KafkaCluster cluster, Map properties) { Properties props = new Properties(); SslPropertiesUtil.addKafkaSslProperties(cluster.getOriginalProperties().getSsl(), props); props.putAll(cluster.getProperties()); props.put(ConsumerConfig.CLIENT_ID_CONFIG, "kafka-ui-consumer-" + System.currentTimeMillis()); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); props.putAll(properties); return new EnhancedConsumer( props, cluster.getPollingSettings().getPollingThrottler(), ApplicationMetrics.forCluster(cluster) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/DeserializationService.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.SerdeDescriptionDTO; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.ClusterSerdes; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import com.provectus.kafka.ui.serdes.ProducerRecordCreator; import com.provectus.kafka.ui.serdes.SerdeInstance; import com.provectus.kafka.ui.serdes.SerdesInitializer; import java.io.Closeable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nullable; import javax.validation.ValidationException; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; /** * Class is responsible for managing serdes for kafka clusters. * NOTE: Since Serde interface is designed to be blocking it is required that DeserializationService * (and all Serde-related code) calls executed within special thread pool (boundedElastic). */ @Component public class DeserializationService implements Closeable { private final Map clusterSerdes = new ConcurrentHashMap<>(); public DeserializationService(Environment env, ClustersStorage clustersStorage, ClustersProperties clustersProperties) { var serdesInitializer = new SerdesInitializer(); for (int i = 0; i < clustersProperties.getClusters().size(); i++) { var clusterProperties = clustersProperties.getClusters().get(i); var cluster = clustersStorage.getClusterByName(clusterProperties.getName()).get(); clusterSerdes.put(cluster.getName(), serdesInitializer.init(env, clustersProperties, i)); } } private ClusterSerdes getSerdesFor(KafkaCluster cluster) { return clusterSerdes.get(cluster.getName()); } private Serde.Serializer getSerializer(KafkaCluster cluster, String topic, Serde.Target type, String serdeName) { var serdes = getSerdesFor(cluster); var serde = serdes.serdeForName(serdeName) .orElseThrow(() -> new ValidationException( String.format("Serde %s not found", serdeName))); if (!serde.canSerialize(topic, type)) { throw new ValidationException( String.format("Serde %s can't be applied for '%s' topic's %s serialization", serde, topic, type)); } return serde.serializer(topic, type); } private SerdeInstance getSerdeForDeserialize(KafkaCluster cluster, String topic, Serde.Target type, @Nullable String serdeName) { var serdes = getSerdesFor(cluster); if (serdeName != null) { var serde = serdes.serdeForName(serdeName) .orElseThrow(() -> new ValidationException(String.format("Serde '%s' not found", serdeName))); if (!serde.canDeserialize(topic, type)) { throw new ValidationException( String.format("Serde '%s' can't be applied to '%s' topic %s", serdeName, topic, type)); } return serde; } else { return serdes.suggestSerdeForDeserialize(topic, type); } } public ProducerRecordCreator producerRecordCreator(KafkaCluster cluster, String topic, String keySerdeName, String valueSerdeName) { return new ProducerRecordCreator( getSerializer(cluster, topic, Serde.Target.KEY, keySerdeName), getSerializer(cluster, topic, Serde.Target.VALUE, valueSerdeName) ); } public ConsumerRecordDeserializer deserializerFor(KafkaCluster cluster, String topic, @Nullable String keySerdeName, @Nullable String valueSerdeName) { var keySerde = getSerdeForDeserialize(cluster, topic, Serde.Target.KEY, keySerdeName); var valueSerde = getSerdeForDeserialize(cluster, topic, Serde.Target.VALUE, valueSerdeName); var fallbackSerde = getSerdesFor(cluster).getFallbackSerde(); return new ConsumerRecordDeserializer( keySerde.getName(), keySerde.deserializer(topic, Serde.Target.KEY), valueSerde.getName(), valueSerde.deserializer(topic, Serde.Target.VALUE), fallbackSerde.getName(), fallbackSerde.deserializer(topic, Serde.Target.KEY), fallbackSerde.deserializer(topic, Serde.Target.VALUE), cluster.getMasking().getMaskerForTopic(topic) ); } public List getSerdesForSerialize(KafkaCluster cluster, String topic, Serde.Target serdeType) { var serdes = getSerdesFor(cluster); var preferred = serdes.suggestSerdeForSerialize(topic, serdeType); var result = new ArrayList(); result.add(toDto(preferred, topic, serdeType, true)); serdes.all() .filter(s -> !s.getName().equals(preferred.getName())) .filter(s -> s.canSerialize(topic, serdeType)) .forEach(s -> result.add(toDto(s, topic, serdeType, false))); return result; } public List getSerdesForDeserialize(KafkaCluster cluster, String topic, Serde.Target serdeType) { var serdes = getSerdesFor(cluster); var preferred = serdes.suggestSerdeForDeserialize(topic, serdeType); var result = new ArrayList(); result.add(toDto(preferred, topic, serdeType, true)); serdes.all() .filter(s -> !s.getName().equals(preferred.getName())) .filter(s -> s.canDeserialize(topic, serdeType)) .forEach(s -> result.add(toDto(s, topic, serdeType, false))); return result; } private SerdeDescriptionDTO toDto(SerdeInstance serdeInstance, String topic, Serde.Target serdeType, boolean preferred) { var schemaOpt = serdeInstance.getSchema(topic, serdeType); return new SerdeDescriptionDTO() .name(serdeInstance.getName()) .description(serdeInstance.description().orElse(null)) .schema(schemaOpt.map(SchemaDescription::getSchema).orElse(null)) .additionalProperties(schemaOpt.map(SchemaDescription::getAdditionalProperties).orElse(null)) .preferred(preferred); } @Override public void close() { clusterSerdes.values().forEach(ClusterSerdes::close); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/FeatureService.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.ReactiveAdminClient.ClusterDescription; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.acl.AclOperation; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Service @Slf4j public class FeatureService { public Mono> getAvailableFeatures(ReactiveAdminClient adminClient, KafkaCluster cluster, ClusterDescription clusterDescription) { List> features = new ArrayList<>(); if (Optional.ofNullable(cluster.getConnectsClients()) .filter(Predicate.not(Map::isEmpty)) .isPresent()) { features.add(Mono.just(ClusterFeature.KAFKA_CONNECT)); } if (cluster.getKsqlClient() != null) { features.add(Mono.just(ClusterFeature.KSQL_DB)); } if (cluster.getSchemaRegistryClient() != null) { features.add(Mono.just(ClusterFeature.SCHEMA_REGISTRY)); } features.add(topicDeletionEnabled(adminClient)); features.add(aclView(adminClient)); features.add(aclEdit(adminClient, clusterDescription)); return Flux.fromIterable(features).flatMap(m -> m).collectList(); } private Mono topicDeletionEnabled(ReactiveAdminClient adminClient) { return adminClient.isTopicDeletionEnabled() ? Mono.just(ClusterFeature.TOPIC_DELETION) : Mono.empty(); } private Mono aclEdit(ReactiveAdminClient adminClient, ClusterDescription clusterDescription) { var authorizedOps = Optional.ofNullable(clusterDescription.getAuthorizedOperations()).orElse(Set.of()); boolean canEdit = aclViewEnabled(adminClient) && (authorizedOps.contains(AclOperation.ALL) || authorizedOps.contains(AclOperation.ALTER)); return canEdit ? Mono.just(ClusterFeature.KAFKA_ACL_EDIT) : Mono.empty(); } private Mono aclView(ReactiveAdminClient adminClient) { return aclViewEnabled(adminClient) ? Mono.just(ClusterFeature.KAFKA_ACL_VIEW) : Mono.empty(); } private boolean aclViewEnabled(ReactiveAdminClient adminClient) { return adminClient.getClusterFeatures().contains(ReactiveAdminClient.SupportedFeature.AUTHORIZED_SECURITY_ENABLED); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaClusterFactory.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.client.RetryingKafkaConnectClient; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.config.WebclientProperties; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.emitter.PollingSettings; import com.provectus.kafka.ui.model.ApplicationPropertyValidationDTO; import com.provectus.kafka.ui.model.ClusterConfigValidationDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.MetricsConfig; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import com.provectus.kafka.ui.service.masking.DataMasking; import com.provectus.kafka.ui.sr.ApiClient; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.util.KafkaServicesValidation; import com.provectus.kafka.ui.util.ReactiveFailover; import com.provectus.kafka.ui.util.WebClientConfigurator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Service @Slf4j public class KafkaClusterFactory { private static final DataSize DEFAULT_WEBCLIENT_BUFFER = DataSize.parse("20MB"); private final DataSize webClientMaxBuffSize; public KafkaClusterFactory(WebclientProperties webclientProperties) { this.webClientMaxBuffSize = Optional.ofNullable(webclientProperties.getMaxInMemoryBufferSize()) .map(DataSize::parse) .orElse(DEFAULT_WEBCLIENT_BUFFER); } public KafkaCluster create(ClustersProperties properties, ClustersProperties.Cluster clusterProperties) { KafkaCluster.KafkaClusterBuilder builder = KafkaCluster.builder(); builder.name(clusterProperties.getName()); builder.bootstrapServers(clusterProperties.getBootstrapServers()); builder.properties(convertProperties(clusterProperties.getProperties())); builder.readOnly(clusterProperties.isReadOnly()); builder.masking(DataMasking.create(clusterProperties.getMasking())); builder.pollingSettings(PollingSettings.create(clusterProperties, properties)); if (schemaRegistryConfigured(clusterProperties)) { builder.schemaRegistryClient(schemaRegistryClient(clusterProperties)); } if (connectClientsConfigured(clusterProperties)) { builder.connectsClients(connectClients(clusterProperties)); } if (ksqlConfigured(clusterProperties)) { builder.ksqlClient(ksqlClient(clusterProperties)); } if (metricsConfigured(clusterProperties)) { builder.metricsConfig(metricsConfigDataToMetricsConfig(clusterProperties.getMetrics())); } builder.originalProperties(clusterProperties); return builder.build(); } public Mono validate(ClustersProperties.Cluster clusterProperties) { if (clusterProperties.getSsl() != null) { Optional errMsg = KafkaServicesValidation.validateTruststore(clusterProperties.getSsl()); if (errMsg.isPresent()) { return Mono.just(new ClusterConfigValidationDTO() .kafka(new ApplicationPropertyValidationDTO() .error(true) .errorMessage("Truststore not valid: " + errMsg.get()))); } } return Mono.zip( KafkaServicesValidation.validateClusterConnection( clusterProperties.getBootstrapServers(), convertProperties(clusterProperties.getProperties()), clusterProperties.getSsl() ), schemaRegistryConfigured(clusterProperties) ? KafkaServicesValidation.validateSchemaRegistry( () -> schemaRegistryClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), ksqlConfigured(clusterProperties) ? KafkaServicesValidation.validateKsql(() -> ksqlClient(clusterProperties)).map(Optional::of) : Mono.>just(Optional.empty()), connectClientsConfigured(clusterProperties) ? Flux.fromIterable(clusterProperties.getKafkaConnect()) .flatMap(c -> KafkaServicesValidation.validateConnect(() -> connectClient(clusterProperties, c)) .map(r -> Tuples.of(c.getName(), r))) .collectMap(Tuple2::getT1, Tuple2::getT2) .map(Optional::of) : Mono.>>just(Optional.empty()) ).map(tuple -> { var validation = new ClusterConfigValidationDTO(); validation.kafka(tuple.getT1()); tuple.getT2().ifPresent(validation::schemaRegistry); tuple.getT3().ifPresent(validation::ksqldb); tuple.getT4().ifPresent(validation::kafkaConnects); return validation; }); } private Properties convertProperties(Map propertiesMap) { Properties properties = new Properties(); if (propertiesMap != null) { properties.putAll(propertiesMap); } return properties; } private boolean connectClientsConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getKafkaConnect() != null; } private Map> connectClients( ClustersProperties.Cluster clusterProperties) { Map> connects = new HashMap<>(); clusterProperties.getKafkaConnect().forEach(c -> connects.put(c.getName(), connectClient(clusterProperties, c))); return connects; } private ReactiveFailover connectClient(ClustersProperties.Cluster cluster, ClustersProperties.ConnectCluster connectCluster) { return ReactiveFailover.create( parseUrlList(connectCluster.getAddress()), url -> new RetryingKafkaConnectClient( connectCluster.toBuilder().address(url).build(), cluster.getSsl(), webClientMaxBuffSize ), ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, "No alive connect instances available", ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS ); } private boolean schemaRegistryConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getSchemaRegistry() != null; } private ReactiveFailover schemaRegistryClient(ClustersProperties.Cluster clusterProperties) { var auth = Optional.ofNullable(clusterProperties.getSchemaRegistryAuth()) .orElse(new ClustersProperties.SchemaRegistryAuth()); WebClient webClient = new WebClientConfigurator() .configureSsl(clusterProperties.getSsl(), clusterProperties.getSchemaRegistrySsl()) .configureBasicAuth(auth.getUsername(), auth.getPassword()) .configureBufferSize(webClientMaxBuffSize) .build(); return ReactiveFailover.create( parseUrlList(clusterProperties.getSchemaRegistry()), url -> new KafkaSrClientApi(new ApiClient(webClient, null, null).setBasePath(url)), ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, "No live schemaRegistry instances available", ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS ); } private boolean ksqlConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getKsqldbServer() != null; } private ReactiveFailover ksqlClient(ClustersProperties.Cluster clusterProperties) { return ReactiveFailover.create( parseUrlList(clusterProperties.getKsqldbServer()), url -> new KsqlApiClient( url, clusterProperties.getKsqldbServerAuth(), clusterProperties.getSsl(), clusterProperties.getKsqldbServerSsl(), webClientMaxBuffSize ), ReactiveFailover.CONNECTION_REFUSED_EXCEPTION_FILTER, "No live ksqldb instances available", ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS ); } private List parseUrlList(String url) { return Stream.of(url.split(",")).map(String::trim).filter(s -> !s.isBlank()).toList(); } private boolean metricsConfigured(ClustersProperties.Cluster clusterProperties) { return clusterProperties.getMetrics() != null; } @Nullable private MetricsConfig metricsConfigDataToMetricsConfig(ClustersProperties.MetricsConfigData metricsConfigData) { if (metricsConfigData == null) { return null; } MetricsConfig.MetricsConfigBuilder builder = MetricsConfig.builder(); builder.type(metricsConfigData.getType()); builder.port(metricsConfigData.getPort()); builder.ssl(Optional.ofNullable(metricsConfigData.getSsl()).orElse(false)); builder.username(metricsConfigData.getUsername()); builder.password(metricsConfigData.getPassword()); builder.keystoreLocation(metricsConfigData.getKeystoreLocation()); builder.keystorePassword(metricsConfigData.getKeystorePassword()); return builder.build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConfigSanitizer.java ================================================ package com.provectus.kafka.ui.service; import static java.util.regex.Pattern.CASE_INSENSITIVE; import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.SslConfigs; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component class KafkaConfigSanitizer { private static final String SANITIZED_VALUE = "******"; private static final String[] REGEX_PARTS = {"*", "$", "^", "+"}; private static final List DEFAULT_PATTERNS_TO_SANITIZE = ImmutableList.builder() .addAll(kafkaConfigKeysToSanitize()) .add( "basic.auth.user.info", /* For Schema Registry credentials */ "password", "secret", "token", "key", ".*credentials.*", /* General credential patterns */ "aws.access.*", "aws.secret.*", "aws.session.*" /* AWS-related credential patterns */ ) .build(); private final List sanitizeKeysPatterns; KafkaConfigSanitizer( @Value("${kafka.config.sanitizer.enabled:true}") boolean enabled, @Value("${kafka.config.sanitizer.patterns:}") List patternsToSanitize ) { this.sanitizeKeysPatterns = enabled ? compile(patternsToSanitize.isEmpty() ? DEFAULT_PATTERNS_TO_SANITIZE : patternsToSanitize) : List.of(); } private static List compile(Collection patternStrings) { return patternStrings.stream() .map(p -> isRegex(p) ? Pattern.compile(p, CASE_INSENSITIVE) : Pattern.compile(".*" + p + "$", CASE_INSENSITIVE)) .toList(); } private static boolean isRegex(String str) { return Arrays.stream(REGEX_PARTS).anyMatch(str::contains); } private static Set kafkaConfigKeysToSanitize() { final ConfigDef configDef = new ConfigDef(); SslConfigs.addClientSslSupport(configDef); SaslConfigs.addClientSaslSupport(configDef); return configDef.configKeys().entrySet().stream() .filter(entry -> entry.getValue().type().equals(ConfigDef.Type.PASSWORD)) .map(Map.Entry::getKey) .collect(Collectors.toSet()); } @Nullable public Object sanitize(String key, @Nullable Object value) { for (Pattern pattern : sanitizeKeysPatterns) { if (pattern.matcher(key).matches()) { return SANITIZED_VALUE; } } return value; } public Map sanitizeConnectorConfig(@Nullable Map original) { var result = new HashMap(); //null-values supporting map! if (original != null) { original.forEach((k, v) -> result.put(k, sanitize(k, v))); } return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java ================================================ package com.provectus.kafka.ui.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.connect.model.ConnectorStatus; import com.provectus.kafka.ui.connect.model.ConnectorStatusConnector; import com.provectus.kafka.ui.connect.model.ConnectorTopics; import com.provectus.kafka.ui.connect.model.TaskStatus; import com.provectus.kafka.ui.exception.NotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.mapper.KafkaConnectMapper; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.ConnectorActionDTO; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigValidationResponseDTO; import com.provectus.kafka.ui.model.ConnectorPluginDTO; import com.provectus.kafka.ui.model.ConnectorStateDTO; import com.provectus.kafka.ui.model.ConnectorTaskStatusDTO; import com.provectus.kafka.ui.model.FullConnectorInfoDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.NewConnectorDTO; import com.provectus.kafka.ui.model.TaskDTO; import com.provectus.kafka.ui.model.connect.InternalConnectInfo; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Service @Slf4j @RequiredArgsConstructor public class KafkaConnectService { private final ClusterMapper clusterMapper; private final KafkaConnectMapper kafkaConnectMapper; private final ObjectMapper objectMapper; private final KafkaConfigSanitizer kafkaConfigSanitizer; public Flux getConnects(KafkaCluster cluster) { return Flux.fromIterable( Optional.ofNullable(cluster.getOriginalProperties().getKafkaConnect()) .map(lst -> lst.stream().map(clusterMapper::toKafkaConnect).toList()) .orElse(List.of()) ); } public Flux getAllConnectors(final KafkaCluster cluster, @Nullable final String search) { return getConnects(cluster) .flatMap(connect -> getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) .flatMap(connectorName -> Mono.zip( getConnector(cluster, connect.getName(), connectorName), getConnectorConfig(cluster, connect.getName(), connectorName), getConnectorTasks(cluster, connect.getName(), connectorName).collectList(), getConnectorTopics(cluster, connect.getName(), connectorName) ).map(tuple -> InternalConnectInfo.builder() .connector(tuple.getT1()) .config(tuple.getT2()) .tasks(tuple.getT3()) .topics(tuple.getT4().getTopics()) .build()))) .map(kafkaConnectMapper::fullConnectorInfo) .filter(matchesSearchTerm(search)); } private Predicate matchesSearchTerm(@Nullable final String search) { if (search == null) { return c -> true; } return connector -> getStringsForSearch(connector) .anyMatch(string -> StringUtils.containsIgnoreCase(string, search)); } private Stream getStringsForSearch(FullConnectorInfoDTO fullConnectorInfo) { return Stream.of( fullConnectorInfo.getName(), fullConnectorInfo.getConnect(), fullConnectorInfo.getStatus().getState().getValue(), fullConnectorInfo.getType().getValue()); } public Mono getConnectorTopics(KafkaCluster cluster, String connectClusterName, String connectorName) { return api(cluster, connectClusterName) .mono(c -> c.getConnectorTopics(connectorName)) .map(result -> result.get(connectorName)) // old Connect API versions don't have this endpoint, setting empty list for // backward-compatibility .onErrorResume(Exception.class, e -> Mono.just(new ConnectorTopics().topics(List.of()))); } public Flux getConnectorNames(KafkaCluster cluster, String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectors(null)) // for some reason `getConnectors` method returns the response as a single string .collectList().map(e -> e.get(0)) .map(this::parseConnectorsNamesStringToList) .flatMapMany(Flux::fromIterable); } // returns empty flux if there was an error communicating with Connect public Flux getConnectorNamesWithErrorsSuppress(KafkaCluster cluster, String connectName) { return getConnectorNames(cluster, connectName).onErrorComplete(); } @SneakyThrows private List parseConnectorsNamesStringToList(String json) { return objectMapper.readValue(json, new TypeReference<>() { }); } public Mono createConnector(KafkaCluster cluster, String connectName, Mono connector) { return api(cluster, connectName) .mono(client -> connector .flatMap(c -> connectorExists(cluster, connectName, c.getName()) .map(exists -> { if (Boolean.TRUE.equals(exists)) { throw new ValidationException( String.format("Connector with name %s already exists", c.getName())); } return c; })) .map(kafkaConnectMapper::toClient) .flatMap(client::createConnector) .flatMap(c -> getConnector(cluster, connectName, c.getName())) ); } private Mono connectorExists(KafkaCluster cluster, String connectName, String connectorName) { return getConnectorNames(cluster, connectName) .any(name -> name.equals(connectorName)); } public Mono getConnector(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) .mono(client -> client.getConnector(connectorName) .map(kafkaConnectMapper::fromClient) .flatMap(connector -> client.getConnectorStatus(connector.getName()) // status request can return 404 if tasks not assigned yet .onErrorResume(WebClientResponseException.NotFound.class, e -> emptyStatus(connectorName)) .map(connectorStatus -> { var status = connectorStatus.getConnector(); var sanitizedConfig = kafkaConfigSanitizer.sanitizeConnectorConfig(connector.getConfig()); ConnectorDTO result = new ConnectorDTO() .connect(connectName) .status(kafkaConnectMapper.fromClient(status)) .type(connector.getType()) .tasks(connector.getTasks()) .name(connector.getName()) .config(sanitizedConfig); if (connectorStatus.getTasks() != null) { boolean isAnyTaskFailed = connectorStatus.getTasks().stream() .map(TaskStatus::getState) .anyMatch(TaskStatus.StateEnum.FAILED::equals); if (isAnyTaskFailed) { result.getStatus().state(ConnectorStateDTO.TASK_FAILED); } } return result; }) ) ); } private Mono emptyStatus(String connectorName) { return Mono.just(new ConnectorStatus() .name(connectorName) .tasks(List.of()) .connector(new ConnectorStatusConnector() .state(ConnectorStatusConnector.StateEnum.UNASSIGNED))); } public Mono> getConnectorConfig(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) .mono(c -> c.getConnectorConfig(connectorName)) .map(kafkaConfigSanitizer::sanitizeConnectorConfig); } public Mono setConnectorConfig(KafkaCluster cluster, String connectName, String connectorName, Mono> requestBody) { return api(cluster, connectName) .mono(c -> requestBody .flatMap(body -> c.setConnectorConfig(connectorName, body)) .map(kafkaConnectMapper::fromClient)); } public Mono deleteConnector( KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) .mono(c -> c.deleteConnector(connectorName)); } public Mono updateConnectorState(KafkaCluster cluster, String connectName, String connectorName, ConnectorActionDTO action) { return api(cluster, connectName) .mono(client -> { switch (action) { case RESTART: return client.restartConnector(connectorName, false, false); case RESTART_ALL_TASKS: return restartTasks(cluster, connectName, connectorName, task -> true); case RESTART_FAILED_TASKS: return restartTasks(cluster, connectName, connectorName, t -> t.getStatus().getState() == ConnectorTaskStatusDTO.FAILED); case PAUSE: return client.pauseConnector(connectorName); case RESUME: return client.resumeConnector(connectorName); default: throw new IllegalStateException("Unexpected value: " + action); } }); } private Mono restartTasks(KafkaCluster cluster, String connectName, String connectorName, Predicate taskFilter) { return getConnectorTasks(cluster, connectName, connectorName) .filter(taskFilter) .flatMap(t -> restartConnectorTask(cluster, connectName, connectorName, t.getId().getTask())) .then(); } public Flux getConnectorTasks(KafkaCluster cluster, String connectName, String connectorName) { return api(cluster, connectName) .flux(client -> client.getConnectorTasks(connectorName) .onErrorResume(WebClientResponseException.NotFound.class, e -> Flux.empty()) .map(kafkaConnectMapper::fromClient) .flatMap(task -> client .getConnectorTaskStatus(connectorName, task.getId().getTask()) .onErrorResume(WebClientResponseException.NotFound.class, e -> Mono.empty()) .map(kafkaConnectMapper::fromClient) .map(task::status) )); } public Mono restartConnectorTask(KafkaCluster cluster, String connectName, String connectorName, Integer taskId) { return api(cluster, connectName) .mono(client -> client.restartConnectorTask(connectorName, taskId)); } public Flux getConnectorPlugins(KafkaCluster cluster, String connectName) { return api(cluster, connectName) .flux(client -> client.getConnectorPlugins().map(kafkaConnectMapper::fromClient)); } public Mono validateConnectorPluginConfig( KafkaCluster cluster, String connectName, String pluginName, Mono> requestBody) { return api(cluster, connectName) .mono(client -> requestBody .flatMap(body -> client.validateConnectorPluginConfig(pluginName, body)) .map(kafkaConnectMapper::fromClient) ); } private ReactiveFailover api(KafkaCluster cluster, String connectName) { var client = cluster.getConnectsClients().get(connectName); if (client == null) { throw new NotFoundException( "Connect %s not found for cluster %s".formatted(connectName, cluster.getName())); } return client; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/MessagesService.java ================================================ package com.provectus.kafka.ui.service; import com.google.common.util.concurrent.RateLimiter; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.emitter.BackwardEmitter; import com.provectus.kafka.ui.emitter.ForwardEmitter; import com.provectus.kafka.ui.emitter.MessageFilters; import com.provectus.kafka.ui.emitter.TailingEmitter; import com.provectus.kafka.ui.exception.TopicNotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.CreateTopicMessageDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.MessageFilterTypeDTO; import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.SmartFilterTestExecutionDTO; import com.provectus.kafka.ui.model.SmartFilterTestExecutionResultDTO; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.ProducerRecordCreator; import com.provectus.kafka.ui.util.SslPropertiesUtil; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @Service @Slf4j public class MessagesService { private static final int DEFAULT_MAX_PAGE_SIZE = 500; private static final int DEFAULT_PAGE_SIZE = 100; // limiting UI messages rate to 20/sec in tailing mode private static final int TAILING_UI_MESSAGE_THROTTLE_RATE = 20; private final AdminClientService adminClientService; private final DeserializationService deserializationService; private final ConsumerGroupService consumerGroupService; private final int maxPageSize; private final int defaultPageSize; public MessagesService(AdminClientService adminClientService, DeserializationService deserializationService, ConsumerGroupService consumerGroupService, ClustersProperties properties) { this.adminClientService = adminClientService; this.deserializationService = deserializationService; this.consumerGroupService = consumerGroupService; var pollingProps = Optional.ofNullable(properties.getPolling()) .orElseGet(ClustersProperties.PollingProperties::new); this.maxPageSize = Optional.ofNullable(pollingProps.getMaxPageSize()) .orElse(DEFAULT_MAX_PAGE_SIZE); this.defaultPageSize = Optional.ofNullable(pollingProps.getDefaultPageSize()) .orElse(DEFAULT_PAGE_SIZE); } private Mono withExistingTopic(KafkaCluster cluster, String topicName) { return adminClientService.get(cluster) .flatMap(client -> client.describeTopic(topicName)) .switchIfEmpty(Mono.error(new TopicNotFoundException())); } public static SmartFilterTestExecutionResultDTO execSmartFilterTest(SmartFilterTestExecutionDTO execData) { Predicate predicate; try { predicate = MessageFilters.createMsgFilter( execData.getFilterCode(), MessageFilterTypeDTO.GROOVY_SCRIPT ); } catch (Exception e) { log.info("Smart filter '{}' compilation error", execData.getFilterCode(), e); return new SmartFilterTestExecutionResultDTO() .error("Compilation error : " + e.getMessage()); } try { var result = predicate.test( new TopicMessageDTO() .key(execData.getKey()) .content(execData.getValue()) .headers(execData.getHeaders()) .offset(execData.getOffset()) .partition(execData.getPartition()) .timestamp( Optional.ofNullable(execData.getTimestampMs()) .map(ts -> OffsetDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC)) .orElse(null)) ); return new SmartFilterTestExecutionResultDTO() .result(result); } catch (Exception e) { log.info("Smart filter {} execution error", execData, e); return new SmartFilterTestExecutionResultDTO() .error("Execution error : " + e.getMessage()); } } public Mono deleteTopicMessages(KafkaCluster cluster, String topicName, List partitionsToInclude) { return withExistingTopic(cluster, topicName) .flatMap(td -> offsetsForDeletion(cluster, topicName, partitionsToInclude) .flatMap(offsets -> adminClientService.get(cluster).flatMap(ac -> ac.deleteRecords(offsets)))); } private Mono> offsetsForDeletion(KafkaCluster cluster, String topicName, List partitionsToInclude) { return adminClientService.get(cluster).flatMap(ac -> ac.listTopicOffsets(topicName, OffsetSpec.earliest(), true) .zipWith(ac.listTopicOffsets(topicName, OffsetSpec.latest(), true), (start, end) -> end.entrySet().stream() .filter(e -> partitionsToInclude.isEmpty() || partitionsToInclude.contains(e.getKey().partition())) // we only need non-empty partitions (where start offset != end offset) .filter(entry -> !entry.getValue().equals(start.get(entry.getKey()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) ); } public Mono sendMessage(KafkaCluster cluster, String topic, CreateTopicMessageDTO msg) { return withExistingTopic(cluster, topic) .publishOn(Schedulers.boundedElastic()) .flatMap(desc -> sendMessageImpl(cluster, desc, msg)); } private Mono sendMessageImpl(KafkaCluster cluster, TopicDescription topicDescription, CreateTopicMessageDTO msg) { if (msg.getPartition() != null && msg.getPartition() > topicDescription.partitions().size() - 1) { return Mono.error(new ValidationException("Invalid partition")); } ProducerRecordCreator producerRecordCreator = deserializationService.producerRecordCreator( cluster, topicDescription.name(), msg.getKeySerde().get(), msg.getValueSerde().get() ); try (KafkaProducer producer = createProducer(cluster, Map.of())) { ProducerRecord producerRecord = producerRecordCreator.create( topicDescription.name(), msg.getPartition(), msg.getKey().orElse(null), msg.getContent().orElse(null), msg.getHeaders() ); CompletableFuture cf = new CompletableFuture<>(); producer.send(producerRecord, (metadata, exception) -> { if (exception != null) { cf.completeExceptionally(exception); } else { cf.complete(metadata); } }); return Mono.fromFuture(cf); } catch (Throwable e) { return Mono.error(e); } } public static KafkaProducer createProducer(KafkaCluster cluster, Map additionalProps) { Properties properties = new Properties(); SslPropertiesUtil.addKafkaSslProperties(cluster.getOriginalProperties().getSsl(), properties); properties.putAll(cluster.getProperties()); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); properties.putAll(additionalProps); return new KafkaProducer<>(properties); } public Flux loadMessages(KafkaCluster cluster, String topic, ConsumerPosition consumerPosition, @Nullable String query, MessageFilterTypeDTO filterQueryType, @Nullable Integer pageSize, SeekDirectionDTO seekDirection, @Nullable String keySerde, @Nullable String valueSerde) { return withExistingTopic(cluster, topic) .flux() .publishOn(Schedulers.boundedElastic()) .flatMap(td -> loadMessagesImpl(cluster, topic, consumerPosition, query, filterQueryType, fixPageSize(pageSize), seekDirection, keySerde, valueSerde)); } private int fixPageSize(@Nullable Integer pageSize) { return Optional.ofNullable(pageSize) .filter(ps -> ps > 0 && ps <= maxPageSize) .orElse(defaultPageSize); } private Flux loadMessagesImpl(KafkaCluster cluster, String topic, ConsumerPosition consumerPosition, @Nullable String query, MessageFilterTypeDTO filterQueryType, int limit, SeekDirectionDTO seekDirection, @Nullable String keySerde, @Nullable String valueSerde) { var deserializer = deserializationService.deserializerFor(cluster, topic, keySerde, valueSerde); var filter = getMsgFilter(query, filterQueryType); var emitter = switch (seekDirection) { case FORWARD -> new ForwardEmitter( () -> consumerGroupService.createConsumer(cluster), consumerPosition, limit, deserializer, filter, cluster.getPollingSettings() ); case BACKWARD -> new BackwardEmitter( () -> consumerGroupService.createConsumer(cluster), consumerPosition, limit, deserializer, filter, cluster.getPollingSettings() ); case TAILING -> new TailingEmitter( () -> consumerGroupService.createConsumer(cluster), consumerPosition, deserializer, filter, cluster.getPollingSettings() ); }; return Flux.create(emitter) .map(throttleUiPublish(seekDirection)); } private Predicate getMsgFilter(String query, MessageFilterTypeDTO filterQueryType) { if (StringUtils.isEmpty(query)) { return evt -> true; } return MessageFilters.createMsgFilter(query, filterQueryType); } private UnaryOperator throttleUiPublish(SeekDirectionDTO seekDirection) { if (seekDirection == SeekDirectionDTO.TAILING) { RateLimiter rateLimiter = RateLimiter.create(TAILING_UI_MESSAGE_THROTTLE_RATE); return m -> { rateLimiter.acquire(1); return m; }; } // there is no need to throttle UI production rate for non-tailing modes, since max number of produced // messages is limited for them (with page size) return UnaryOperator.identity(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/OffsetsResetService.java ================================================ package com.provectus.kafka.ui.service; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.kafka.common.ConsumerGroupState.DEAD; import static org.apache.kafka.common.ConsumerGroupState.EMPTY; import com.google.common.base.Preconditions; import com.provectus.kafka.ui.exception.NotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.KafkaCluster; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.common.TopicPartition; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; /** * Implementation follows https://cwiki.apache.org/confluence/display/KAFKA/KIP-122%3A+Add+Reset+Consumer+Group+Offsets+tooling * to works like "kafka-consumer-groups --reset-offsets" console command * (see kafka.admin.ConsumerGroupCommand) */ @Slf4j @Component @RequiredArgsConstructor public class OffsetsResetService { private final AdminClientService adminClientService; public Mono resetToEarliest( KafkaCluster cluster, String group, String topic, Collection partitions) { return checkGroupCondition(cluster, group) .flatMap(ac -> offsets(ac, topic, partitions, OffsetSpec.earliest()) .flatMap(offsets -> resetOffsets(ac, group, offsets))); } private Mono> offsets(ReactiveAdminClient client, String topic, @Nullable Collection partitions, OffsetSpec spec) { if (partitions == null) { return client.listTopicOffsets(topic, spec, true); } return client.listOffsets( partitions.stream().map(idx -> new TopicPartition(topic, idx)).collect(toSet()), spec, true ); } public Mono resetToLatest( KafkaCluster cluster, String group, String topic, Collection partitions) { return checkGroupCondition(cluster, group) .flatMap(ac -> offsets(ac, topic, partitions, OffsetSpec.latest()) .flatMap(offsets -> resetOffsets(ac, group, offsets))); } public Mono resetToTimestamp( KafkaCluster cluster, String group, String topic, Collection partitions, long targetTimestamp) { return checkGroupCondition(cluster, group) .flatMap(ac -> offsets(ac, topic, partitions, OffsetSpec.forTimestamp(targetTimestamp)) .flatMap( foundOffsets -> offsets(ac, topic, partitions, OffsetSpec.latest()) .map(endOffsets -> editTsOffsets(foundOffsets, endOffsets)) ) .flatMap(offsets -> resetOffsets(ac, group, offsets)) ); } public Mono resetToOffsets( KafkaCluster cluster, String group, String topic, Map targetOffsets) { Preconditions.checkNotNull(targetOffsets); var partitionOffsets = targetOffsets.entrySet().stream() .collect(toMap(e -> new TopicPartition(topic, e.getKey()), Map.Entry::getValue)); return checkGroupCondition(cluster, group).flatMap( ac -> ac.listOffsets(partitionOffsets.keySet(), OffsetSpec.earliest(), true) .flatMap(earliest -> ac.listOffsets(partitionOffsets.keySet(), OffsetSpec.latest(), true) .map(latest -> editOffsetsBounds(partitionOffsets, earliest, latest)) .flatMap(offsetsToCommit -> resetOffsets(ac, group, offsetsToCommit))) ); } private Mono checkGroupCondition(KafkaCluster cluster, String groupId) { return adminClientService.get(cluster) .flatMap(ac -> // we need to call listConsumerGroups() to check group existence, because // describeConsumerGroups() will return consumer group even if it doesn't exist ac.listConsumerGroupNames() .filter(cgs -> cgs.stream().anyMatch(g -> g.equals(groupId))) .flatMap(cgs -> ac.describeConsumerGroups(List.of(groupId))) .filter(cgs -> cgs.containsKey(groupId)) .map(cgs -> cgs.get(groupId)) .flatMap(cg -> { if (!Set.of(DEAD, EMPTY).contains(cg.state())) { return Mono.error( new ValidationException( String.format( "Group's offsets can be reset only if group is inactive," + " but group is in %s state", cg.state() ) ) ); } return Mono.just(ac); }) .switchIfEmpty(Mono.error(new NotFoundException("Consumer group not found"))) ); } private Map editTsOffsets(Map foundTsOffsets, Map endOffsets) { // for partitions where we didnt find offset by timestamp, we use end offsets Map result = new HashMap<>(endOffsets); result.putAll(foundTsOffsets); return result; } /** * Checks if submitted offsets is between earliest and latest offsets. If case of range change * fail we reset offset to either earliest or latest offsets (To follow logic from * kafka.admin.ConsumerGroupCommand.scala) */ private Map editOffsetsBounds(Map offsetsToCheck, Map earliestOffsets, Map latestOffsets) { var result = new HashMap(); offsetsToCheck.forEach((tp, offset) -> { if (earliestOffsets.get(tp) > offset) { log.warn("Offset for partition {} is lower than earliest offset, resetting to earliest", tp); result.put(tp, earliestOffsets.get(tp)); } else if (latestOffsets.get(tp) < offset) { log.warn("Offset for partition {} is greater than latest offset, resetting to latest", tp); result.put(tp, latestOffsets.get(tp)); } else { result.put(tp, offset); } }); return result; } private Mono resetOffsets(ReactiveAdminClient adminClient, String groupId, Map offsets) { return adminClient.alterConsumerGroupOffsets(groupId, offsets); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ReactiveAdminClient.java ================================================ package com.provectus.kafka.ui.service; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Iterables; import com.google.common.collect.Table; import com.provectus.kafka.ui.exception.IllegalEntityStateException; import com.provectus.kafka.ui.exception.NotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.util.KafkaVersion; import com.provectus.kafka.ui.util.annotation.KafkaClientInternalsDependant; import java.io.Closeable; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.ConsumerGroupDescription; import org.apache.kafka.clients.admin.ConsumerGroupListing; import org.apache.kafka.clients.admin.DescribeClusterOptions; import org.apache.kafka.clients.admin.DescribeClusterResult; import org.apache.kafka.clients.admin.DescribeConfigsOptions; import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsSpec; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.ListTopicsOptions; import org.apache.kafka.clients.admin.NewPartitionReassignment; import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.admin.ProducerState; import org.apache.kafka.clients.admin.RecordsToDelete; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.acl.AccessControlEntryFilter; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.ClusterAuthorizationException; import org.apache.kafka.common.errors.GroupIdNotFoundException; import org.apache.kafka.common.errors.GroupNotEmptyException; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.SecurityDisabledException; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.requests.DescribeLogDirsResponse; import org.apache.kafka.common.resource.ResourcePatternFilter; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Slf4j @AllArgsConstructor public class ReactiveAdminClient implements Closeable { public enum SupportedFeature { INCREMENTAL_ALTER_CONFIGS(2.3f), CONFIG_DOCUMENTATION_RETRIEVAL(2.6f), DESCRIBE_CLUSTER_INCLUDE_AUTHORIZED_OPERATIONS(2.3f), AUTHORIZED_SECURITY_ENABLED(ReactiveAdminClient::isAuthorizedSecurityEnabled); private final BiFunction> predicate; SupportedFeature(BiFunction> predicate) { this.predicate = predicate; } SupportedFeature(float fromVersion) { this.predicate = (admin, ver) -> Mono.just(ver != null && ver >= fromVersion); } static Mono> forVersion(AdminClient ac, String kafkaVersionStr) { @Nullable Float kafkaVersion = KafkaVersion.parse(kafkaVersionStr).orElse(null); return Flux.fromArray(SupportedFeature.values()) .flatMap(f -> f.predicate.apply(ac, kafkaVersion).map(enabled -> Tuples.of(f, enabled))) .filter(Tuple2::getT2) .map(Tuple2::getT1) .collect(Collectors.toSet()); } } @Value public static class ClusterDescription { @Nullable Node controller; String clusterId; Collection nodes; @Nullable // null, if ACL is disabled Set authorizedOperations; } @Builder private record ConfigRelatedInfo(String version, Set features, boolean topicDeletionIsAllowed) { static final Duration UPDATE_DURATION = Duration.of(1, ChronoUnit.HOURS); private static Mono extract(AdminClient ac) { return ReactiveAdminClient.describeClusterImpl(ac, Set.of()) .flatMap(desc -> { // choosing node from which we will get configs (starting with controller) var targetNodeId = Optional.ofNullable(desc.controller) .map(Node::id) .orElse(desc.getNodes().iterator().next().id()); return loadBrokersConfig(ac, List.of(targetNodeId)) .map(map -> map.isEmpty() ? List.of() : map.get(targetNodeId)) .flatMap(configs -> { String version = "1.0-UNKNOWN"; boolean topicDeletionEnabled = true; for (ConfigEntry entry : configs) { if (entry.name().contains("inter.broker.protocol.version")) { version = entry.value(); } if (entry.name().equals("delete.topic.enable")) { topicDeletionEnabled = Boolean.parseBoolean(entry.value()); } } final String finalVersion = version; final boolean finalTopicDeletionEnabled = topicDeletionEnabled; return SupportedFeature.forVersion(ac, version) .map(features -> new ConfigRelatedInfo(finalVersion, features, finalTopicDeletionEnabled)); }); }) .cache(UPDATE_DURATION); } } public static Mono create(AdminClient adminClient) { Mono configRelatedInfoMono = ConfigRelatedInfo.extract(adminClient); return configRelatedInfoMono.map(info -> new ReactiveAdminClient(adminClient, configRelatedInfoMono, info)); } private static Mono isAuthorizedSecurityEnabled(AdminClient ac, @Nullable Float kafkaVersion) { return toMono(ac.describeAcls(AclBindingFilter.ANY).values()) .thenReturn(true) .doOnError(th -> !(th instanceof SecurityDisabledException) && !(th instanceof InvalidRequestException) && !(th instanceof UnsupportedVersionException), th -> log.debug("Error checking if security enabled", th)) .onErrorReturn(false); } // NOTE: if KafkaFuture returns null, that Mono will be empty(!), since Reactor does not support nullable results // (see MonoSink.success(..) javadoc for details) public static Mono toMono(KafkaFuture future) { return Mono.create(sink -> future.whenComplete((res, ex) -> { if (ex != null) { // KafkaFuture doc is unclear about what exception wrapper will be used // (from docs it should be ExecutionException, be we actually see CompletionException, so checking both if (ex instanceof CompletionException || ex instanceof ExecutionException) { sink.error(ex.getCause()); //unwrapping exception } else { sink.error(ex); } } else { sink.success(res); } })).doOnCancel(() -> future.cancel(true)) // AdminClient is using single thread for kafka communication // and by default all downstream operations (like map(..)) on created Mono will be executed on this thread. // If some of downstream operation are blocking (by mistake) this can lead to // other AdminClient's requests stucking, which can cause timeout exceptions. // So, we explicitly setting Scheduler for downstream processing. .publishOn(Schedulers.parallel()); } //--------------------------------------------------------------------------------- @Getter(AccessLevel.PACKAGE) // visible for testing private final AdminClient client; private final Mono configRelatedInfoMono; private volatile ConfigRelatedInfo configRelatedInfo; public Set getClusterFeatures() { return configRelatedInfo.features(); } public Mono> listTopics(boolean listInternal) { return toMono(client.listTopics(new ListTopicsOptions().listInternal(listInternal)).names()); } public Mono deleteTopic(String topicName) { return toMono(client.deleteTopics(List.of(topicName)).all()); } public String getVersion() { return configRelatedInfo.version(); } public boolean isTopicDeletionEnabled() { return configRelatedInfo.topicDeletionIsAllowed(); } public Mono updateInternalStats(@Nullable Node controller) { if (controller == null) { return Mono.empty(); } return configRelatedInfoMono .doOnNext(info -> this.configRelatedInfo = info) .then(); } public Mono>> getTopicsConfig() { return listTopics(true).flatMap(topics -> getTopicsConfig(topics, false)); } //NOTE: skips not-found topics (for which UnknownTopicOrPartitionException was thrown by AdminClient) //and topics for which DESCRIBE_CONFIGS permission is not set (TopicAuthorizationException was thrown) public Mono>> getTopicsConfig(Collection topicNames, boolean includeDoc) { var includeDocFixed = includeDoc && getClusterFeatures().contains(SupportedFeature.CONFIG_DOCUMENTATION_RETRIEVAL); // we need to partition calls, because it can lead to AdminClient timeouts in case of large topics count return partitionCalls( topicNames, 200, part -> getTopicsConfigImpl(part, includeDocFixed), mapMerger() ); } private Mono>> getTopicsConfigImpl(Collection topicNames, boolean includeDoc) { List resources = topicNames.stream() .map(topicName -> new ConfigResource(ConfigResource.Type.TOPIC, topicName)) .collect(toList()); return toMonoWithExceptionFilter( client.describeConfigs( resources, new DescribeConfigsOptions().includeSynonyms(true).includeDocumentation(includeDoc)).values(), UnknownTopicOrPartitionException.class, TopicAuthorizationException.class ).map(config -> config.entrySet().stream() .collect(toMap( c -> c.getKey().name(), c -> List.copyOf(c.getValue().entries())))); } private static Mono>> loadBrokersConfig(AdminClient client, List brokerIds) { List resources = brokerIds.stream() .map(brokerId -> new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(brokerId))) .collect(toList()); return toMono(client.describeConfigs(resources).all()) // some kafka backends don't support broker's configs retrieval, // and throw various exceptions on describeConfigs() call .onErrorResume(th -> th instanceof InvalidRequestException // MSK Serverless || th instanceof UnknownTopicOrPartitionException, // Azure event hub th -> { log.trace("Error while getting configs for brokers {}", brokerIds, th); return Mono.just(Map.of()); }) // there are situations when kafka-ui user has no DESCRIBE_CONFIGS permission on cluster .onErrorResume(ClusterAuthorizationException.class, th -> { log.trace("AuthorizationException while getting configs for brokers {}", brokerIds, th); return Mono.just(Map.of()); }) // catching all remaining exceptions, but logging on WARN level .onErrorResume(th -> true, th -> { log.warn("Unexpected error while getting configs for brokers {}", brokerIds, th); return Mono.just(Map.of()); }) .map(config -> config.entrySet().stream() .collect(toMap( c -> Integer.valueOf(c.getKey().name()), c -> new ArrayList<>(c.getValue().entries())))); } /** * Return per-broker configs or empty map if broker's configs retrieval not supported. */ public Mono>> loadBrokersConfig(List brokerIds) { return loadBrokersConfig(client, brokerIds); } public Mono> describeTopics() { return listTopics(true).flatMap(this::describeTopics); } public Mono> describeTopics(Collection topics) { // we need to partition calls, because it can lead to AdminClient timeouts in case of large topics count return partitionCalls( topics, 200, this::describeTopicsImpl, mapMerger() ); } private Mono> describeTopicsImpl(Collection topics) { return toMonoWithExceptionFilter( client.describeTopics(topics).topicNameValues(), UnknownTopicOrPartitionException.class, // we only describe topics that we see from listTopics() API, so we should have permission to do it, // but also adding this exception here for rare case when access restricted after we called listTopics() TopicAuthorizationException.class ); } /** * Returns TopicDescription mono, or Empty Mono if topic not visible. */ public Mono describeTopic(String topic) { return describeTopics(List.of(topic)).flatMap(m -> Mono.justOrEmpty(m.get(topic))); } /** * Kafka API often returns Map responses with KafkaFuture values. If we do allOf() * logic resulting Mono will be failing if any of Futures finished with error. * In some situations it is not what we want, ex. we call describeTopics(List names) method and * we getting UnknownTopicOrPartitionException for unknown topics and we what to just not put * such topics in resulting map. *

* This method converts input map into Mono[Map] ignoring keys for which KafkaFutures * finished with classes exceptions and empty Monos. */ @SafeVarargs static Mono> toMonoWithExceptionFilter(Map> values, Class... classes) { if (values.isEmpty()) { return Mono.just(Map.of()); } List>>> monos = values.entrySet().stream() .map(e -> toMono(e.getValue()) .map(r -> Tuples.of(e.getKey(), Optional.of(r))) .defaultIfEmpty(Tuples.of(e.getKey(), Optional.empty())) //tracking empty Monos .onErrorResume( // tracking Monos with suppressible error th -> Stream.of(classes).anyMatch(clazz -> th.getClass().isAssignableFrom(clazz)), th -> Mono.just(Tuples.of(e.getKey(), Optional.empty())))) .toList(); return Mono.zip( monos, resultsArr -> Stream.of(resultsArr) .map(obj -> (Tuple2>) obj) .filter(t -> t.getT2().isPresent()) //skipping empty & suppressible-errors .collect(Collectors.toMap(Tuple2::getT1, t -> t.getT2().get())) ); } public Mono>> describeLogDirs() { return describeCluster() .map(d -> d.getNodes().stream().map(Node::id).collect(toList())) .flatMap(this::describeLogDirs); } public Mono>> describeLogDirs( Collection brokerIds) { return toMono(client.describeLogDirs(brokerIds).all()) .onErrorResume(UnsupportedVersionException.class, th -> Mono.just(Map.of())) .onErrorResume(ClusterAuthorizationException.class, th -> Mono.just(Map.of())) .onErrorResume(th -> true, th -> { log.warn("Error while calling describeLogDirs", th); return Mono.just(Map.of()); }); } public Mono describeCluster() { return describeClusterImpl(client, getClusterFeatures()); } private static Mono describeClusterImpl(AdminClient client, Set features) { boolean includeAuthorizedOperations = features.contains(SupportedFeature.DESCRIBE_CLUSTER_INCLUDE_AUTHORIZED_OPERATIONS); DescribeClusterResult result = client.describeCluster( new DescribeClusterOptions().includeAuthorizedOperations(includeAuthorizedOperations)); var allOfFuture = KafkaFuture.allOf( result.controller(), result.clusterId(), result.nodes(), result.authorizedOperations()); return toMono(allOfFuture).then( Mono.fromCallable(() -> new ClusterDescription( result.controller().get(), result.clusterId().get(), result.nodes().get(), result.authorizedOperations().get() ) ) ); } public Mono deleteConsumerGroups(Collection groupIds) { return toMono(client.deleteConsumerGroups(groupIds).all()) .onErrorResume(GroupIdNotFoundException.class, th -> Mono.error(new NotFoundException("The group id does not exist"))) .onErrorResume(GroupNotEmptyException.class, th -> Mono.error(new IllegalEntityStateException("The group is not empty"))); } public Mono createTopic(String name, int numPartitions, @Nullable Integer replicationFactor, Map configs) { var newTopic = new NewTopic( name, Optional.of(numPartitions), Optional.ofNullable(replicationFactor).map(Integer::shortValue) ).configs(configs); return toMono(client.createTopics(List.of(newTopic)).all()); } public Mono alterPartitionReassignments( Map> reassignments) { return toMono(client.alterPartitionReassignments(reassignments).all()); } public Mono createPartitions(Map newPartitionsMap) { return toMono(client.createPartitions(newPartitionsMap).all()); } // NOTE: places whole current topic config with new one. Entries that were present in old config, // but missed in new will be set to default public Mono updateTopicConfig(String topicName, Map configs) { if (getClusterFeatures().contains(SupportedFeature.INCREMENTAL_ALTER_CONFIGS)) { return getTopicsConfigImpl(List.of(topicName), false) .map(conf -> conf.getOrDefault(topicName, List.of())) .flatMap(currentConfigs -> incrementalAlterConfig(topicName, currentConfigs, configs)); } else { return alterConfig(topicName, configs); } } public Mono> listConsumerGroupNames() { return listConsumerGroups().map(lst -> lst.stream().map(ConsumerGroupListing::groupId).toList()); } public Mono> listConsumerGroups() { return toMono(client.listConsumerGroups().all()); } public Mono> describeConsumerGroups(Collection groupIds) { return partitionCalls( groupIds, 25, 4, ids -> toMono(client.describeConsumerGroups(ids).all()), mapMerger() ); } // group -> partition -> offset // NOTE: partitions with no committed offsets will be skipped public Mono> listConsumerGroupOffsets(List consumerGroups, // all partitions if null passed @Nullable List partitions) { Function, Mono>>> call = groups -> toMono( client.listConsumerGroupOffsets( groups.stream() .collect(Collectors.toMap( g -> g, g -> new ListConsumerGroupOffsetsSpec().topicPartitions(partitions) ))).all() ); Mono>> merged = partitionCalls( consumerGroups, 25, 4, call, mapMerger() ); return merged.map(map -> { var table = ImmutableTable.builder(); map.forEach((g, tpOffsets) -> tpOffsets.forEach((tp, offset) -> { if (offset != null) { // offset will be null for partitions that don't have committed offset for this group table.put(g, tp, offset.offset()); } })); return table.build(); }); } public Mono alterConsumerGroupOffsets(String groupId, Map offsets) { return toMono(client.alterConsumerGroupOffsets( groupId, offsets.entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> new OffsetAndMetadata(e.getValue())))) .all()); } /** * List offset for the topic's partitions and OffsetSpec. * * @param failOnUnknownLeader true - throw exception in case of no-leader partitions, * false - skip partitions with no leader */ public Mono> listTopicOffsets(String topic, OffsetSpec offsetSpec, boolean failOnUnknownLeader) { return describeTopic(topic) .map(td -> filterPartitionsWithLeaderCheck(List.of(td), p -> true, failOnUnknownLeader)) .flatMap(partitions -> listOffsetsUnsafe(partitions, offsetSpec)); } /** * List offset for the specified partitions and OffsetSpec. * * @param failOnUnknownLeader true - throw exception in case of no-leader partitions, * false - skip partitions with no leader */ public Mono> listOffsets(Collection partitions, OffsetSpec offsetSpec, boolean failOnUnknownLeader) { return filterPartitionsWithLeaderCheck(partitions, failOnUnknownLeader) .flatMap(parts -> listOffsetsUnsafe(parts, offsetSpec)); } /** * List offset for the specified topics, skipping no-leader partitions. */ public Mono> listOffsets(Collection topicDescriptions, OffsetSpec offsetSpec) { return listOffsetsUnsafe(filterPartitionsWithLeaderCheck(topicDescriptions, p -> true, false), offsetSpec); } private Mono> filterPartitionsWithLeaderCheck(Collection partitions, boolean failOnUnknownLeader) { var targetTopics = partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet()); return describeTopicsImpl(targetTopics) .map(descriptions -> filterPartitionsWithLeaderCheck( descriptions.values(), partitions::contains, failOnUnknownLeader)); } @VisibleForTesting static Set filterPartitionsWithLeaderCheck(Collection topicDescriptions, Predicate partitionPredicate, boolean failOnUnknownLeader) { var goodPartitions = new HashSet(); for (TopicDescription description : topicDescriptions) { var goodTopicPartitions = new ArrayList(); for (TopicPartitionInfo partitionInfo : description.partitions()) { TopicPartition topicPartition = new TopicPartition(description.name(), partitionInfo.partition()); if (partitionInfo.leader() == null) { if (failOnUnknownLeader) { throw new ValidationException(String.format("Topic partition %s has no leader", topicPartition)); } else { // if ANY of topic partitions has no leader - we have to skip all topic partitions goodTopicPartitions.clear(); break; } } if (partitionPredicate.test(topicPartition)) { goodTopicPartitions.add(topicPartition); } } goodPartitions.addAll(goodTopicPartitions); } return goodPartitions; } // 1. NOTE(!): should only apply for partitions from topics where all partitions have leaders, // otherwise AdminClient will try to fetch topic metadata, fail and retry infinitely (until timeout) // 2. NOTE(!): Skips partitions that were not initialized yet // (UnknownTopicOrPartitionException thrown, ex. after topic creation) // 3. TODO: check if it is a bug that AdminClient never throws LeaderNotAvailableException and just retrying instead @KafkaClientInternalsDependant @VisibleForTesting Mono> listOffsetsUnsafe(Collection partitions, OffsetSpec offsetSpec) { if (partitions.isEmpty()) { return Mono.just(Map.of()); } Function, Mono>> call = parts -> { ListOffsetsResult r = client.listOffsets(parts.stream().collect(toMap(tp -> tp, tp -> offsetSpec))); Map> perPartitionResults = new HashMap<>(); parts.forEach(p -> perPartitionResults.put(p, r.partitionResult(p))); return toMonoWithExceptionFilter(perPartitionResults, UnknownTopicOrPartitionException.class) .map(offsets -> offsets.entrySet().stream() // filtering partitions for which offsets were not found .filter(e -> e.getValue().offset() >= 0) .collect(toMap(Map.Entry::getKey, e -> e.getValue().offset()))); }; return partitionCalls( partitions, 200, call, mapMerger() ); } public Mono> listAcls(ResourcePatternFilter filter) { Preconditions.checkArgument(getClusterFeatures().contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); return toMono(client.describeAcls(new AclBindingFilter(filter, AccessControlEntryFilter.ANY)).values()); } public Mono createAcls(Collection aclBindings) { Preconditions.checkArgument(getClusterFeatures().contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); return toMono(client.createAcls(aclBindings).all()); } public Mono deleteAcls(Collection aclBindings) { Preconditions.checkArgument(getClusterFeatures().contains(SupportedFeature.AUTHORIZED_SECURITY_ENABLED)); var filters = aclBindings.stream().map(AclBinding::toFilter).collect(Collectors.toSet()); return toMono(client.deleteAcls(filters).all()).then(); } public Mono updateBrokerConfigByName(Integer brokerId, String name, String value) { ConfigResource cr = new ConfigResource(ConfigResource.Type.BROKER, String.valueOf(brokerId)); AlterConfigOp op = new AlterConfigOp(new ConfigEntry(name, value), AlterConfigOp.OpType.SET); return toMono(client.incrementalAlterConfigs(Map.of(cr, List.of(op))).all()); } public Mono deleteRecords(Map offsets) { var records = offsets.entrySet().stream() .map(entry -> Map.entry(entry.getKey(), RecordsToDelete.beforeOffset(entry.getValue()))) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); return toMono(client.deleteRecords(records).all()); } public Mono alterReplicaLogDirs(Map replicaAssignment) { return toMono(client.alterReplicaLogDirs(replicaAssignment).all()); } // returns tp -> list of active producer's states (if any) public Mono>> getActiveProducersState(String topic) { return describeTopic(topic) .map(td -> client.describeProducers( IntStream.range(0, td.partitions().size()) .mapToObj(i -> new TopicPartition(topic, i)) .toList() ).all() ) .flatMap(ReactiveAdminClient::toMono) .map(map -> map.entrySet().stream() .filter(e -> !e.getValue().activeProducers().isEmpty()) // skipping partitions without producers .collect(toMap(Map.Entry::getKey, e -> e.getValue().activeProducers()))); } private Mono incrementalAlterConfig(String topicName, List currentConfigs, Map newConfigs) { var configsToDelete = currentConfigs.stream() .filter(e -> e.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG) //manually set configs only .filter(e -> !newConfigs.containsKey(e.name())) .map(e -> new AlterConfigOp(e, AlterConfigOp.OpType.DELETE)); var configsToSet = newConfigs.entrySet().stream() .map(e -> new AlterConfigOp(new ConfigEntry(e.getKey(), e.getValue()), AlterConfigOp.OpType.SET)); return toMono(client.incrementalAlterConfigs( Map.of( new ConfigResource(ConfigResource.Type.TOPIC, topicName), Stream.concat(configsToDelete, configsToSet).toList() )).all()); } @SuppressWarnings("deprecation") private Mono alterConfig(String topicName, Map configs) { List configEntries = configs.entrySet().stream() .flatMap(cfg -> Stream.of(new ConfigEntry(cfg.getKey(), cfg.getValue()))) .collect(toList()); Config config = new Config(configEntries); var topicResource = new ConfigResource(ConfigResource.Type.TOPIC, topicName); return toMono(client.alterConfigs(Map.of(topicResource, config)).all()); } /** * Splits input collection into batches, converts each batch into Mono, sequentially subscribes to them * and merges output Monos into one Mono. */ private static Mono partitionCalls(Collection items, int partitionSize, Function, Mono> call, BiFunction merger) { if (items.isEmpty()) { return call.apply(items); } Iterable> parts = Iterables.partition(items, partitionSize); return Flux.fromIterable(parts) .concatMap(call) .reduce(merger); } /** * Splits input collection into batches, converts each batch into Mono, subscribes to them (concurrently, * with specified concurrency level) and merges output Monos into one Mono. */ private static Mono partitionCalls(Collection items, int partitionSize, int concurrency, Function, Mono> call, BiFunction merger) { if (items.isEmpty()) { return call.apply(items); } Iterable> parts = Iterables.partition(items, partitionSize); return Flux.fromIterable(parts) .flatMap(call, concurrency) .reduce(merger); } private static BiFunction, Map, Map> mapMerger() { return (m1, m2) -> { var merged = new HashMap(); merged.putAll(m1); merged.putAll(m2); return merged; }; } @Override public void close() { client.close(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/SchemaRegistryService.java ================================================ package com.provectus.kafka.ui.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.exception.SchemaCompatibilityException; import com.provectus.kafka.ui.exception.SchemaNotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.sr.model.Compatibility; import com.provectus.kafka.ui.sr.model.CompatibilityCheckResponse; import com.provectus.kafka.ui.sr.model.CompatibilityConfig; import com.provectus.kafka.ui.sr.model.CompatibilityLevelChange; import com.provectus.kafka.ui.sr.model.NewSubject; import com.provectus.kafka.ui.sr.model.SchemaSubject; import com.provectus.kafka.ui.util.ReactiveFailover; import java.nio.charset.Charset; import java.util.List; import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Service @Slf4j @RequiredArgsConstructor public class SchemaRegistryService { private static final String LATEST = "latest"; @AllArgsConstructor public static class SubjectWithCompatibilityLevel { @Delegate SchemaSubject subject; @Getter Compatibility compatibility; } private ReactiveFailover api(KafkaCluster cluster) { return cluster.getSchemaRegistryClient(); } public Mono> getAllLatestVersionSchemas(KafkaCluster cluster, List subjects) { return Flux.fromIterable(subjects) .concatMap(subject -> getLatestSchemaVersionBySubject(cluster, subject)) .collect(Collectors.toList()); } public Mono> getAllSubjectNames(KafkaCluster cluster) { return api(cluster) .mono(c -> c.getAllSubjectNames(null, false)) .flatMapIterable(this::parseSubjectListString) .collectList(); } @SneakyThrows private List parseSubjectListString(String subjectNamesStr) { //workaround for https://github.com/spring-projects/spring-framework/issues/24734 return new JsonMapper().readValue(subjectNamesStr, new TypeReference>() { }); } public Flux getAllVersionsBySubject(KafkaCluster cluster, String subject) { Flux versions = getSubjectVersions(cluster, subject); return versions.flatMap(version -> getSchemaSubjectByVersion(cluster, subject, version)); } private Flux getSubjectVersions(KafkaCluster cluster, String schemaName) { return api(cluster).flux(c -> c.getSubjectVersions(schemaName)); } public Mono getSchemaSubjectByVersion(KafkaCluster cluster, String schemaName, Integer version) { return getSchemaSubject(cluster, schemaName, String.valueOf(version)); } public Mono getLatestSchemaVersionBySubject(KafkaCluster cluster, String schemaName) { return getSchemaSubject(cluster, schemaName, LATEST); } private Mono getSchemaSubject(KafkaCluster cluster, String schemaName, String version) { return api(cluster) .mono(c -> c.getSubjectVersion(schemaName, version, false)) .zipWith(getSchemaCompatibilityInfoOrGlobal(cluster, schemaName)) .map(t -> new SubjectWithCompatibilityLevel(t.getT1(), t.getT2())) .onErrorResume(WebClientResponseException.NotFound.class, th -> Mono.error(new SchemaNotFoundException())); } public Mono deleteSchemaSubjectByVersion(KafkaCluster cluster, String schemaName, Integer version) { return deleteSchemaSubject(cluster, schemaName, String.valueOf(version)); } public Mono deleteLatestSchemaSubject(KafkaCluster cluster, String schemaName) { return deleteSchemaSubject(cluster, schemaName, LATEST); } private Mono deleteSchemaSubject(KafkaCluster cluster, String schemaName, String version) { return api(cluster).mono(c -> c.deleteSubjectVersion(schemaName, version, false)); } public Mono deleteSchemaSubjectEntirely(KafkaCluster cluster, String schemaName) { return api(cluster).mono(c -> c.deleteAllSubjectVersions(schemaName, false)); } /** * Checks whether the provided schema duplicates the previous or not, creates a new schema * and then returns the whole content by requesting its latest version. */ public Mono registerNewSchema(KafkaCluster cluster, String subject, NewSubject newSchemaSubject) { return api(cluster) .mono(c -> c.registerNewSchema(subject, newSchemaSubject)) .onErrorMap(WebClientResponseException.Conflict.class, th -> new SchemaCompatibilityException()) .onErrorMap(WebClientResponseException.UnprocessableEntity.class, th -> new ValidationException("Invalid schema. Error from registry: " + th.getResponseBodyAsString())) .then(getLatestSchemaVersionBySubject(cluster, subject)); } public Mono updateSchemaCompatibility(KafkaCluster cluster, String schemaName, Compatibility compatibility) { return api(cluster) .mono(c -> c.updateSubjectCompatibilityLevel( schemaName, new CompatibilityLevelChange().compatibility(compatibility))) .then(); } public Mono updateGlobalSchemaCompatibility(KafkaCluster cluster, Compatibility compatibility) { return api(cluster) .mono(c -> c.updateGlobalCompatibilityLevel(new CompatibilityLevelChange().compatibility(compatibility))) .then(); } public Mono getSchemaCompatibilityLevel(KafkaCluster cluster, String schemaName) { return api(cluster) .mono(c -> c.getSubjectCompatibilityLevel(schemaName, true)) .map(CompatibilityConfig::getCompatibilityLevel) .onErrorResume(error -> Mono.empty()); } public Mono getGlobalSchemaCompatibilityLevel(KafkaCluster cluster) { return api(cluster) .mono(KafkaSrClientApi::getGlobalCompatibilityLevel) .map(CompatibilityConfig::getCompatibilityLevel); } private Mono getSchemaCompatibilityInfoOrGlobal(KafkaCluster cluster, String schemaName) { return getSchemaCompatibilityLevel(cluster, schemaName) .switchIfEmpty(this.getGlobalSchemaCompatibilityLevel(cluster)); } public Mono checksSchemaCompatibility(KafkaCluster cluster, String schemaName, NewSubject newSchemaSubject) { return api(cluster).mono(c -> c.checkSchemaCompatibility(schemaName, LATEST, true, newSchemaSubject)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsCache.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.ServerStatusDTO; import com.provectus.kafka.ui.model.Statistics; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.springframework.stereotype.Component; @Component public class StatisticsCache { private final Map cache = new ConcurrentHashMap<>(); public StatisticsCache(ClustersStorage clustersStorage) { var initializing = Statistics.empty().toBuilder().status(ServerStatusDTO.INITIALIZING).build(); clustersStorage.getKafkaClusters().forEach(c -> cache.put(c.getName(), initializing)); } public synchronized void replace(KafkaCluster c, Statistics stats) { cache.put(c.getName(), stats); } public synchronized void update(KafkaCluster c, Map descriptions, Map> configs) { var metrics = get(c); var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); updatedDescriptions.putAll(descriptions); var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); updatedConfigs.putAll(configs); replace( c, metrics.toBuilder() .topicDescriptions(updatedDescriptions) .topicConfigs(updatedConfigs) .build() ); } public synchronized void onTopicDelete(KafkaCluster c, String topic) { var metrics = get(c); var updatedDescriptions = new HashMap<>(metrics.getTopicDescriptions()); updatedDescriptions.remove(topic); var updatedConfigs = new HashMap<>(metrics.getTopicConfigs()); updatedConfigs.remove(topic); replace( c, metrics.toBuilder() .topicDescriptions(updatedDescriptions) .topicConfigs(updatedConfigs) .build() ); } public Statistics get(KafkaCluster c) { return Objects.requireNonNull(cache.get(c.getName()), "Unknown cluster metrics requested"); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/StatisticsService.java ================================================ package com.provectus.kafka.ui.service; import static com.provectus.kafka.ui.service.ReactiveAdminClient.ClusterDescription; import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.InternalLogDirStats; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.ServerStatusDTO; import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.service.metrics.MetricsCollector; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; @Service @RequiredArgsConstructor @Slf4j public class StatisticsService { private final MetricsCollector metricsCollector; private final AdminClientService adminClientService; private final FeatureService featureService; private final StatisticsCache cache; public Mono updateCache(KafkaCluster c) { return getStatistics(c).doOnSuccess(m -> cache.replace(c, m)); } private Mono getStatistics(KafkaCluster cluster) { return adminClientService.get(cluster).flatMap(ac -> ac.describeCluster().flatMap(description -> ac.updateInternalStats(description.getController()).then( Mono.zip( List.of( metricsCollector.getBrokerMetrics(cluster, description.getNodes()), getLogDirInfo(description, ac), featureService.getAvailableFeatures(ac, cluster, description), loadTopicConfigs(cluster), describeTopics(cluster)), results -> Statistics.builder() .status(ServerStatusDTO.ONLINE) .clusterDescription(description) .version(ac.getVersion()) .metrics((Metrics) results[0]) .logDirInfo((InternalLogDirStats) results[1]) .features((List) results[2]) .topicConfigs((Map>) results[3]) .topicDescriptions((Map) results[4]) .build() )))) .doOnError(e -> log.error("Failed to collect cluster {} info", cluster.getName(), e)) .onErrorResume( e -> Mono.just(Statistics.empty().toBuilder().lastKafkaException(e).build())); } private Mono getLogDirInfo(ClusterDescription desc, ReactiveAdminClient ac) { var brokerIds = desc.getNodes().stream().map(Node::id).collect(Collectors.toSet()); return ac.describeLogDirs(brokerIds).map(InternalLogDirStats::new); } private Mono> describeTopics(KafkaCluster c) { return adminClientService.get(c).flatMap(ReactiveAdminClient::describeTopics); } private Mono>> loadTopicConfigs(KafkaCluster c) { return adminClientService.get(c).flatMap(ReactiveAdminClient::getTopicsConfig); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/TopicsService.java ================================================ package com.provectus.kafka.ui.service; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import com.google.common.collect.Sets; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.exception.TopicMetadataException; import com.provectus.kafka.ui.exception.TopicNotFoundException; import com.provectus.kafka.ui.exception.TopicRecreationException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.ClusterFeature; import com.provectus.kafka.ui.model.InternalLogDirStats; import com.provectus.kafka.ui.model.InternalPartition; import com.provectus.kafka.ui.model.InternalPartitionsOffsets; import com.provectus.kafka.ui.model.InternalReplica; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.InternalTopicConfig; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.PartitionsIncreaseDTO; import com.provectus.kafka.ui.model.PartitionsIncreaseResponseDTO; import com.provectus.kafka.ui.model.ReplicationFactorChangeDTO; import com.provectus.kafka.ui.model.ReplicationFactorChangeResponseDTO; import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.TopicUpdateDTO; import java.time.Duration; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.NewPartitionReassignment; import org.apache.kafka.clients.admin.NewPartitions; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.admin.ProducerState; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TopicExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; import reactor.util.retry.Retry; @Service @RequiredArgsConstructor public class TopicsService { private final AdminClientService adminClientService; private final StatisticsCache statisticsCache; private final ClustersProperties clustersProperties; @Value("${topic.recreate.maxRetries:15}") private int recreateMaxRetries; @Value("${topic.recreate.delay.seconds:1}") private int recreateDelayInSeconds; @Value("${topic.load.after.create.maxRetries:10}") private int loadTopicAfterCreateRetries; @Value("${topic.load.after.create.delay.ms:500}") private int loadTopicAfterCreateDelayInMs; public Mono> loadTopics(KafkaCluster c, List topics) { if (topics.isEmpty()) { return Mono.just(List.of()); } return adminClientService.get(c) .flatMap(ac -> ac.describeTopics(topics).zipWith(ac.getTopicsConfig(topics, false), (descriptions, configs) -> { statisticsCache.update(c, descriptions, configs); return getPartitionOffsets(descriptions, ac).map(offsets -> { var metrics = statisticsCache.get(c); return createList( topics, descriptions, configs, offsets, metrics.getMetrics(), metrics.getLogDirInfo() ); }); })).flatMap(Function.identity()); } private Mono loadTopic(KafkaCluster c, String topicName) { return loadTopics(c, List.of(topicName)) .flatMap(lst -> lst.stream().findFirst() .map(Mono::just) .orElse(Mono.error(TopicNotFoundException::new))); } /** * After creation topic can be invisible via API for some time. * To workaround this, we retyring topic loading until it becomes visible. */ private Mono loadTopicAfterCreation(KafkaCluster c, String topicName) { return loadTopic(c, topicName) .retryWhen( Retry .fixedDelay( loadTopicAfterCreateRetries, Duration.ofMillis(loadTopicAfterCreateDelayInMs) ) .filter(TopicNotFoundException.class::isInstance) .onRetryExhaustedThrow((spec, sig) -> new TopicMetadataException( String.format( "Error while loading created topic '%s' - topic is not visible via API " + "after waiting for %d ms.", topicName, loadTopicAfterCreateDelayInMs * loadTopicAfterCreateRetries))) ); } private List createList(List orderedNames, Map descriptions, Map> configs, InternalPartitionsOffsets partitionsOffsets, Metrics metrics, InternalLogDirStats logDirInfo) { return orderedNames.stream() .filter(descriptions::containsKey) .map(t -> InternalTopic.from( descriptions.get(t), configs.getOrDefault(t, List.of()), partitionsOffsets, metrics, logDirInfo, clustersProperties.getInternalTopicPrefix() )) .collect(toList()); } private Mono getPartitionOffsets(Map descriptionsMap, ReactiveAdminClient ac) { var descriptions = descriptionsMap.values(); return ac.listOffsets(descriptions, OffsetSpec.earliest()) .zipWith(ac.listOffsets(descriptions, OffsetSpec.latest()), (earliest, latest) -> Sets.intersection(earliest.keySet(), latest.keySet()) .stream() .map(tp -> Map.entry(tp, new InternalPartitionsOffsets.Offsets( earliest.get(tp), latest.get(tp)))) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))) .map(InternalPartitionsOffsets::new); } public Mono getTopicDetails(KafkaCluster cluster, String topicName) { return loadTopic(cluster, topicName); } public Mono> getTopicConfigs(KafkaCluster cluster, String topicName) { // there 2 case that we cover here: // 1. topic not found/visible - describeTopic() will be empty and we will throw TopicNotFoundException // 2. topic is visible, but we don't have DESCRIBE_CONFIG permission - we should return empty list return adminClientService.get(cluster) .flatMap(ac -> ac.describeTopic(topicName) .switchIfEmpty(Mono.error(new TopicNotFoundException())) .then(ac.getTopicsConfig(List.of(topicName), true)) .map(m -> m.values().stream().findFirst().orElse(List.of()))); } private Mono createTopic(KafkaCluster c, ReactiveAdminClient adminClient, TopicCreationDTO topicData) { return adminClient.createTopic( topicData.getName(), topicData.getPartitions(), topicData.getReplicationFactor(), topicData.getConfigs()) .thenReturn(topicData) .onErrorMap(t -> new TopicMetadataException(t.getMessage(), t)) .then(loadTopicAfterCreation(c, topicData.getName())); } public Mono createTopic(KafkaCluster cluster, TopicCreationDTO topicCreation) { return adminClientService.get(cluster) .flatMap(ac -> createTopic(cluster, ac, topicCreation)); } public Mono recreateTopic(KafkaCluster cluster, String topicName) { return loadTopic(cluster, topicName) .flatMap(t -> deleteTopic(cluster, topicName) .thenReturn(t) .delayElement(Duration.ofSeconds(recreateDelayInSeconds)) .flatMap(topic -> adminClientService.get(cluster) .flatMap(ac -> ac.createTopic( topic.getName(), topic.getPartitionCount(), topic.getReplicationFactor(), topic.getTopicConfigs() .stream() .collect(Collectors.toMap(InternalTopicConfig::getName, InternalTopicConfig::getValue)) ) .thenReturn(topicName) ) .retryWhen( Retry.fixedDelay(recreateMaxRetries, Duration.ofSeconds(recreateDelayInSeconds)) .filter(TopicExistsException.class::isInstance) .onRetryExhaustedThrow((a, b) -> new TopicRecreationException(topicName, recreateMaxRetries * recreateDelayInSeconds)) ) .flatMap(a -> loadTopicAfterCreation(cluster, topicName)) ) ); } private Mono updateTopic(KafkaCluster cluster, String topicName, TopicUpdateDTO topicUpdate) { return adminClientService.get(cluster) .flatMap(ac -> ac.updateTopicConfig(topicName, topicUpdate.getConfigs()) .then(loadTopic(cluster, topicName))); } public Mono updateTopic(KafkaCluster cl, String topicName, Mono topicUpdate) { return topicUpdate .flatMap(t -> updateTopic(cl, topicName, t)); } private Mono changeReplicationFactor( KafkaCluster cluster, ReactiveAdminClient adminClient, String topicName, Map> reassignments ) { return adminClient.alterPartitionReassignments(reassignments) .then(loadTopic(cluster, topicName)); } /** * Change topic replication factor, works on brokers versions 5.4.x and higher */ public Mono changeReplicationFactor( KafkaCluster cluster, String topicName, ReplicationFactorChangeDTO replicationFactorChange) { return loadTopic(cluster, topicName).flatMap(topic -> adminClientService.get(cluster) .flatMap(ac -> { Integer actual = topic.getReplicationFactor(); Integer requested = replicationFactorChange.getTotalReplicationFactor(); Integer brokersCount = statisticsCache.get(cluster).getClusterDescription() .getNodes().size(); if (requested.equals(actual)) { return Mono.error( new ValidationException( String.format("Topic already has replicationFactor %s.", actual))); } if (requested <= 0) { return Mono.error( new ValidationException( String.format("Requested replication factor (%s) should be greater or equal to 1.", requested))); } if (requested > brokersCount) { return Mono.error( new ValidationException( String.format("Requested replication factor %s more than brokers count %s.", requested, brokersCount))); } return changeReplicationFactor(cluster, ac, topicName, getPartitionsReassignments(cluster, topic, replicationFactorChange)); }) .map(t -> new ReplicationFactorChangeResponseDTO() .topicName(t.getName()) .totalReplicationFactor(t.getReplicationFactor()))); } private Map> getPartitionsReassignments( KafkaCluster cluster, InternalTopic topic, ReplicationFactorChangeDTO replicationFactorChange) { // Current assignment map (Partition number -> List of brokers) Map> currentAssignment = getCurrentAssignment(topic); // Brokers map (Broker id -> count) Map brokersUsage = getBrokersMap(cluster, currentAssignment); int currentReplicationFactor = topic.getReplicationFactor(); // If we should to increase Replication factor if (replicationFactorChange.getTotalReplicationFactor() > currentReplicationFactor) { // For each partition for (var assignmentList : currentAssignment.values()) { // Get brokers list sorted by usage var brokers = brokersUsage.entrySet().stream() .sorted(Map.Entry.comparingByValue()) .map(Map.Entry::getKey) .collect(toList()); // Iterate brokers and try to add them in assignment // while partition replicas count != requested replication factor for (Integer broker : brokers) { if (!assignmentList.contains(broker)) { assignmentList.add(broker); brokersUsage.merge(broker, 1, Integer::sum); } if (assignmentList.size() == replicationFactorChange.getTotalReplicationFactor()) { break; } } if (assignmentList.size() != replicationFactorChange.getTotalReplicationFactor()) { throw new ValidationException("Something went wrong during adding replicas"); } } // If we should to decrease Replication factor } else if (replicationFactorChange.getTotalReplicationFactor() < currentReplicationFactor) { for (Map.Entry> assignmentEntry : currentAssignment.entrySet()) { var partition = assignmentEntry.getKey(); var brokers = assignmentEntry.getValue(); // Get brokers list sorted by usage in reverse order var brokersUsageList = brokersUsage.entrySet().stream() .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) .map(Map.Entry::getKey) .collect(toList()); // Iterate brokers and try to remove them from assignment // while partition replicas count != requested replication factor for (Integer broker : brokersUsageList) { // Check is the broker the leader of partition if (!topic.getPartitions().get(partition).getLeader() .equals(broker)) { brokers.remove(broker); brokersUsage.merge(broker, -1, Integer::sum); } if (brokers.size() == replicationFactorChange.getTotalReplicationFactor()) { break; } } if (brokers.size() != replicationFactorChange.getTotalReplicationFactor()) { throw new ValidationException("Something went wrong during removing replicas"); } } } else { throw new ValidationException("Replication factor already equals requested"); } // Return result map return currentAssignment.entrySet().stream().collect(toMap( e -> new TopicPartition(topic.getName(), e.getKey()), e -> Optional.of(new NewPartitionReassignment(e.getValue())) )); } private Map> getCurrentAssignment(InternalTopic topic) { return topic.getPartitions().values().stream() .collect(toMap( InternalPartition::getPartition, p -> p.getReplicas().stream() .map(InternalReplica::getBroker) .collect(toList()) )); } private Map getBrokersMap(KafkaCluster cluster, Map> currentAssignment) { Map result = statisticsCache.get(cluster).getClusterDescription().getNodes() .stream() .map(Node::id) .collect(toMap( c -> c, c -> 0 )); currentAssignment.values().forEach(brokers -> brokers .forEach(broker -> result.put(broker, result.get(broker) + 1))); return result; } public Mono increaseTopicPartitions( KafkaCluster cluster, String topicName, PartitionsIncreaseDTO partitionsIncrease) { return loadTopic(cluster, topicName).flatMap(topic -> adminClientService.get(cluster).flatMap(ac -> { Integer actualCount = topic.getPartitionCount(); Integer requestedCount = partitionsIncrease.getTotalPartitionsCount(); if (requestedCount < actualCount) { return Mono.error( new ValidationException(String.format( "Topic currently has %s partitions, which is higher than the requested %s.", actualCount, requestedCount))); } if (requestedCount.equals(actualCount)) { return Mono.error( new ValidationException( String.format("Topic already has %s partitions.", actualCount))); } Map newPartitionsMap = Collections.singletonMap( topicName, NewPartitions.increaseTo(partitionsIncrease.getTotalPartitionsCount()) ); return ac.createPartitions(newPartitionsMap) .then(loadTopic(cluster, topicName)); }).map(t -> new PartitionsIncreaseResponseDTO() .topicName(t.getName()) .totalPartitionsCount(t.getPartitionCount()) ) ); } public Mono deleteTopic(KafkaCluster cluster, String topicName) { if (statisticsCache.get(cluster).getFeatures().contains(ClusterFeature.TOPIC_DELETION)) { return adminClientService.get(cluster).flatMap(c -> c.deleteTopic(topicName)) .doOnSuccess(t -> statisticsCache.onTopicDelete(cluster, topicName)); } else { return Mono.error(new ValidationException("Topic deletion restricted")); } } public Mono cloneTopic( KafkaCluster cluster, String topicName, String newTopicName) { return loadTopic(cluster, topicName).flatMap(topic -> adminClientService.get(cluster) .flatMap(ac -> ac.createTopic( newTopicName, topic.getPartitionCount(), topic.getReplicationFactor(), topic.getTopicConfigs() .stream() .collect(Collectors .toMap(InternalTopicConfig::getName, InternalTopicConfig::getValue)) ) ).thenReturn(newTopicName) .flatMap(a -> loadTopicAfterCreation(cluster, newTopicName)) ); } public Mono> getTopicsForPagination(KafkaCluster cluster) { Statistics stats = statisticsCache.get(cluster); return filterExisting(cluster, stats.getTopicDescriptions().keySet()) .map(lst -> lst.stream() .map(topicName -> InternalTopic.from( stats.getTopicDescriptions().get(topicName), stats.getTopicConfigs().getOrDefault(topicName, List.of()), InternalPartitionsOffsets.empty(), stats.getMetrics(), stats.getLogDirInfo(), clustersProperties.getInternalTopicPrefix() )) .collect(toList()) ); } public Mono>> getActiveProducersState(KafkaCluster cluster, String topic) { return adminClientService.get(cluster) .flatMap(ac -> ac.getActiveProducersState(topic)); } private Mono> filterExisting(KafkaCluster cluster, Collection topics) { return adminClientService.get(cluster) .flatMap(ac -> ac.listTopics(true)) .map(existing -> existing .stream() .filter(topics::contains) .collect(toList())); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclCsv.java ================================================ package com.provectus.kafka.ui.service.acl; import com.provectus.kafka.ui.exception.ValidationException; import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; public class AclCsv { private static final String LINE_SEPARATOR = System.lineSeparator(); private static final String VALUES_SEPARATOR = ","; private static final String HEADER = "Principal,ResourceType,PatternType,ResourceName,Operation,PermissionType,Host"; public static String transformToCsvString(Collection acls) { return Stream.concat(Stream.of(HEADER), acls.stream().map(AclCsv::createAclString)) .collect(Collectors.joining(System.lineSeparator())); } public static String createAclString(AclBinding binding) { var pattern = binding.pattern(); var filter = binding.toFilter().entryFilter(); return String.format( "%s,%s,%s,%s,%s,%s,%s", filter.principal(), pattern.resourceType(), pattern.patternType(), pattern.name(), filter.operation(), filter.permissionType(), filter.host() ); } private static AclBinding parseCsvLine(String csv, int line) { String[] values = csv.split(VALUES_SEPARATOR); if (values.length != 7) { throw new ValidationException("Input csv is not valid - there should be 7 columns in line " + line); } for (int i = 0; i < values.length; i++) { if ((values[i] = values[i].trim()).isBlank()) { throw new ValidationException("Input csv is not valid - blank value in colum " + i + ", line " + line); } } try { return new AclBinding( new ResourcePattern( ResourceType.valueOf(values[1]), values[3], PatternType.valueOf(values[2])), new AccessControlEntry( values[0], values[6], AclOperation.valueOf(values[4]), AclPermissionType.valueOf(values[5])) ); } catch (IllegalArgumentException enumParseError) { throw new ValidationException("Error parsing enum value in line " + line); } } public static Collection parseCsv(String csvString) { String[] lines = csvString.split(LINE_SEPARATOR); if (lines.length == 0) { throw new ValidationException("Error parsing ACL csv file: no lines in file"); } boolean firstLineIsHeader = HEADER.equalsIgnoreCase(lines[0].trim().replace(" ", "")); Set result = new HashSet<>(); for (int i = firstLineIsHeader ? 1 : 0; i < lines.length; i++) { String line = lines[i]; if (!line.isBlank()) { AclBinding aclBinding = parseCsvLine(line, i); result.add(aclBinding); } } return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/acl/AclsService.java ================================================ package com.provectus.kafka.ui.service.acl; import static org.apache.kafka.common.acl.AclOperation.ALL; import static org.apache.kafka.common.acl.AclOperation.CREATE; import static org.apache.kafka.common.acl.AclOperation.DESCRIBE; import static org.apache.kafka.common.acl.AclOperation.IDEMPOTENT_WRITE; import static org.apache.kafka.common.acl.AclOperation.READ; import static org.apache.kafka.common.acl.AclOperation.WRITE; import static org.apache.kafka.common.acl.AclPermissionType.ALLOW; import static org.apache.kafka.common.resource.PatternType.LITERAL; import static org.apache.kafka.common.resource.PatternType.PREFIXED; import static org.apache.kafka.common.resource.ResourceType.CLUSTER; import static org.apache.kafka.common.resource.ResourceType.GROUP; import static org.apache.kafka.common.resource.ResourceType.TOPIC; import static org.apache.kafka.common.resource.ResourceType.TRANSACTIONAL_ID; import com.google.common.collect.Sets; import com.provectus.kafka.ui.model.CreateConsumerAclDTO; import com.provectus.kafka.ui.model.CreateProducerAclDTO; import com.provectus.kafka.ui.model.CreateStreamAppAclDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.AdminClientService; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.resource.Resource; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j @Service @RequiredArgsConstructor public class AclsService { private final AdminClientService adminClientService; public Mono createAcl(KafkaCluster cluster, AclBinding aclBinding) { return adminClientService.get(cluster) .flatMap(ac -> createAclsWithLogging(ac, List.of(aclBinding))); } private Mono createAclsWithLogging(ReactiveAdminClient ac, Collection bindings) { bindings.forEach(b -> log.info("CREATING ACL: [{}]", AclCsv.createAclString(b))); return ac.createAcls(bindings) .doOnSuccess(v -> bindings.forEach(b -> log.info("ACL CREATED: [{}]", AclCsv.createAclString(b)))); } public Mono deleteAcl(KafkaCluster cluster, AclBinding aclBinding) { var aclString = AclCsv.createAclString(aclBinding); log.info("DELETING ACL: [{}]", aclString); return adminClientService.get(cluster) .flatMap(ac -> ac.deleteAcls(List.of(aclBinding))) .doOnSuccess(v -> log.info("ACL DELETED: [{}]", aclString)); } public Flux listAcls(KafkaCluster cluster, ResourcePatternFilter filter) { return adminClientService.get(cluster) .flatMap(c -> c.listAcls(filter)) .flatMapIterable(acls -> acls) .sort(Comparator.comparing(AclBinding::toString)); //sorting to keep stable order on different calls } public Mono getAclAsCsvString(KafkaCluster cluster) { return adminClientService.get(cluster) .flatMap(c -> c.listAcls(ResourcePatternFilter.ANY)) .map(AclCsv::transformToCsvString); } public Mono syncAclWithAclCsv(KafkaCluster cluster, String csv) { return adminClientService.get(cluster) .flatMap(ac -> ac.listAcls(ResourcePatternFilter.ANY).flatMap(existingAclList -> { var existingSet = Set.copyOf(existingAclList); var newAcls = Set.copyOf(AclCsv.parseCsv(csv)); var toDelete = Sets.difference(existingSet, newAcls); var toAdd = Sets.difference(newAcls, existingSet); logAclSyncPlan(cluster, toAdd, toDelete); if (toAdd.isEmpty() && toDelete.isEmpty()) { return Mono.empty(); } log.info("Starting new ACLs creation"); return ac.createAcls(toAdd) .doOnSuccess(v -> { log.info("{} new ACLs created", toAdd.size()); log.info("Starting ACLs deletion"); }) .then(ac.deleteAcls(toDelete) .doOnSuccess(v -> log.info("{} ACLs deleted", toDelete.size()))); })); } private void logAclSyncPlan(KafkaCluster cluster, Set toBeAdded, Set toBeDeleted) { log.info("'{}' cluster ACL sync plan: ", cluster.getName()); if (toBeAdded.isEmpty() && toBeDeleted.isEmpty()) { log.info("Nothing to do, ACL is already in sync"); return; } if (!toBeAdded.isEmpty()) { log.info("ACLs to be added ({}): ", toBeAdded.size()); for (AclBinding aclBinding : toBeAdded) { log.info(" " + AclCsv.createAclString(aclBinding)); } } if (!toBeDeleted.isEmpty()) { log.info("ACLs to be deleted ({}): ", toBeDeleted.size()); for (AclBinding aclBinding : toBeDeleted) { log.info(" " + AclCsv.createAclString(aclBinding)); } } } // creates allow binding for resources by prefix or specific names list private List createAllowBindings(ResourceType resourceType, List opsToAllow, String principal, String host, @Nullable String resourcePrefix, @Nullable Collection resourceNames) { List bindings = new ArrayList<>(); if (resourcePrefix != null) { for (var op : opsToAllow) { bindings.add( new AclBinding( new ResourcePattern(resourceType, resourcePrefix, PREFIXED), new AccessControlEntry(principal, host, op, ALLOW))); } } if (!CollectionUtils.isEmpty(resourceNames)) { resourceNames.stream() .distinct() .forEach(resource -> opsToAllow.forEach(op -> bindings.add( new AclBinding( new ResourcePattern(resourceType, resource, LITERAL), new AccessControlEntry(principal, host, op, ALLOW))))); } return bindings; } public Mono createConsumerAcl(KafkaCluster cluster, CreateConsumerAclDTO request) { return adminClientService.get(cluster) .flatMap(ac -> createAclsWithLogging(ac, createConsumerBindings(request))) .then(); } //Read, Describe on topics, Read on consumerGroups private List createConsumerBindings(CreateConsumerAclDTO request) { List bindings = new ArrayList<>(); bindings.addAll( createAllowBindings(TOPIC, List.of(READ, DESCRIBE), request.getPrincipal(), request.getHost(), request.getTopicsPrefix(), request.getTopics())); bindings.addAll( createAllowBindings( GROUP, List.of(READ), request.getPrincipal(), request.getHost(), request.getConsumerGroupsPrefix(), request.getConsumerGroups())); return bindings; } public Mono createProducerAcl(KafkaCluster cluster, CreateProducerAclDTO request) { return adminClientService.get(cluster) .flatMap(ac -> createAclsWithLogging(ac, createProducerBindings(request))) .then(); } //Write, Describe, Create permission on topics, Write, Describe on transactionalIds //IDEMPOTENT_WRITE on cluster if idempotent is enabled private List createProducerBindings(CreateProducerAclDTO request) { List bindings = new ArrayList<>(); bindings.addAll( createAllowBindings( TOPIC, List.of(WRITE, DESCRIBE, CREATE), request.getPrincipal(), request.getHost(), request.getTopicsPrefix(), request.getTopics())); bindings.addAll( createAllowBindings( TRANSACTIONAL_ID, List.of(WRITE, DESCRIBE), request.getPrincipal(), request.getHost(), request.getTransactionsIdPrefix(), Optional.ofNullable(request.getTransactionalId()).map(List::of).orElse(null))); if (Boolean.TRUE.equals(request.getIdempotent())) { bindings.addAll( createAllowBindings( CLUSTER, List.of(IDEMPOTENT_WRITE), request.getPrincipal(), request.getHost(), null, List.of(Resource.CLUSTER_NAME))); // cluster name is a const string in ACL api } return bindings; } public Mono createStreamAppAcl(KafkaCluster cluster, CreateStreamAppAclDTO request) { return adminClientService.get(cluster) .flatMap(ac -> createAclsWithLogging(ac, createStreamAppBindings(request))) .then(); } // Read on input topics, Write on output topics // ALL on applicationId-prefixed Groups and Topics private List createStreamAppBindings(CreateStreamAppAclDTO request) { List bindings = new ArrayList<>(); bindings.addAll( createAllowBindings( TOPIC, List.of(READ), request.getPrincipal(), request.getHost(), null, request.getInputTopics())); bindings.addAll( createAllowBindings( TOPIC, List.of(WRITE), request.getPrincipal(), request.getHost(), null, request.getOutputTopics())); bindings.addAll( createAllowBindings( GROUP, List.of(ALL), request.getPrincipal(), request.getHost(), request.getApplicationId(), null)); bindings.addAll( createAllowBindings( TOPIC, List.of(ALL), request.getPrincipal(), request.getHost(), request.getApplicationId(), null)); return bindings; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/AnalysisTasksStore.java ================================================ package com.provectus.kafka.ui.service.analyze; import com.google.common.base.Throwables; import com.provectus.kafka.ui.model.TopicAnalysisDTO; import com.provectus.kafka.ui.model.TopicAnalysisProgressDTO; import com.provectus.kafka.ui.model.TopicAnalysisResultDTO; import java.io.Closeable; import java.math.BigDecimal; import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.Builder; import lombok.SneakyThrows; import lombok.Value; class AnalysisTasksStore { private final Map running = new ConcurrentHashMap<>(); private final Map completed = new ConcurrentHashMap<>(); void setAnalysisError(TopicIdentity topicId, Instant collectionStartedAt, Throwable th) { running.remove(topicId); completed.put( topicId, new TopicAnalysisResultDTO() .startedAt(collectionStartedAt.toEpochMilli()) .finishedAt(System.currentTimeMillis()) .error(Throwables.getStackTraceAsString(th)) ); } void setAnalysisResult(TopicIdentity topicId, Instant collectionStartedAt, TopicAnalysisStats totalStats, Map partitionStats) { running.remove(topicId); completed.put(topicId, new TopicAnalysisResultDTO() .startedAt(collectionStartedAt.toEpochMilli()) .finishedAt(System.currentTimeMillis()) .totalStats(totalStats.toDto(null)) .partitionStats( partitionStats.entrySet().stream() .map(e -> e.getValue().toDto(e.getKey())) .collect(Collectors.toList()) )); } void updateProgress(TopicIdentity topicId, long msgsScanned, long bytesScanned, Double completeness) { running.computeIfPresent(topicId, (k, state) -> state.toBuilder() .msgsScanned(msgsScanned) .bytesScanned(bytesScanned) .completenessPercent(completeness) .build()); } void registerNewTask(TopicIdentity topicId, Closeable task) { running.put(topicId, new RunningAnalysis(Instant.now(), 0.0, 0, 0, task)); } void cancelAnalysis(TopicIdentity topicId) { Optional.ofNullable(running.remove(topicId)) .ifPresent(RunningAnalysis::stopTask); } boolean isAnalysisInProgress(TopicIdentity id) { return running.containsKey(id); } Optional getTopicAnalysis(TopicIdentity id) { var runningState = running.get(id); var completedState = completed.get(id); if (runningState == null && completedState == null) { return Optional.empty(); } return Optional.of(createAnalysisDto(runningState, completedState)); } private TopicAnalysisDTO createAnalysisDto(@Nullable RunningAnalysis runningState, @Nullable TopicAnalysisResultDTO completedState) { return new TopicAnalysisDTO() .progress(runningState != null ? runningState.toDto() : null) .result(completedState); } @Builder(toBuilder = true) private record RunningAnalysis(Instant startedAt, double completenessPercent, long msgsScanned, long bytesScanned, Closeable task) { TopicAnalysisProgressDTO toDto() { return new TopicAnalysisProgressDTO() .startedAt(startedAt.toEpochMilli()) .bytesScanned(bytesScanned) .msgsScanned(msgsScanned) .completenessPercent(BigDecimal.valueOf(completenessPercent)); } @SneakyThrows void stopTask() { task.close(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisService.java ================================================ package com.provectus.kafka.ui.service.analyze; import static com.provectus.kafka.ui.model.SeekTypeDTO.BEGINNING; import com.provectus.kafka.ui.emitter.EnhancedConsumer; import com.provectus.kafka.ui.emitter.SeekOperations; import com.provectus.kafka.ui.exception.TopicAnalysisException; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.TopicAnalysisDTO; import com.provectus.kafka.ui.service.ConsumerGroupService; import com.provectus.kafka.ui.service.TopicsService; import java.io.Closeable; import java.time.Duration; import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; @Slf4j @Component @RequiredArgsConstructor public class TopicAnalysisService { private static final Scheduler SCHEDULER = Schedulers.newBoundedElastic( Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, "topic-analysis-tasks", 10, //ttl for idle threads (in sec) true //daemon ); private final AnalysisTasksStore analysisTasksStore = new AnalysisTasksStore(); private final TopicsService topicsService; private final ConsumerGroupService consumerGroupService; public Mono analyze(KafkaCluster cluster, String topicName) { return topicsService.getTopicDetails(cluster, topicName) .doOnNext(topic -> startAnalysis(cluster, topicName)) .then(); } private synchronized void startAnalysis(KafkaCluster cluster, String topic) { var topicId = new TopicIdentity(cluster, topic); if (analysisTasksStore.isAnalysisInProgress(topicId)) { throw new TopicAnalysisException("Topic is already analyzing"); } var task = new AnalysisTask(cluster, topicId); analysisTasksStore.registerNewTask(topicId, task); SCHEDULER.schedule(task); } public void cancelAnalysis(KafkaCluster cluster, String topicName) { analysisTasksStore.cancelAnalysis(new TopicIdentity(cluster, topicName)); } public Optional getTopicAnalysis(KafkaCluster cluster, String topicName) { return analysisTasksStore.getTopicAnalysis(new TopicIdentity(cluster, topicName)); } class AnalysisTask implements Runnable, Closeable { private final Instant startedAt = Instant.now(); private final TopicIdentity topicId; private final TopicAnalysisStats totalStats = new TopicAnalysisStats(); private final Map partitionStats = new HashMap<>(); private final EnhancedConsumer consumer; AnalysisTask(KafkaCluster cluster, TopicIdentity topicId) { this.topicId = topicId; this.consumer = consumerGroupService.createConsumer( cluster, // to improve polling throughput Map.of( ConsumerConfig.RECEIVE_BUFFER_CONFIG, "-1", //let OS tune buffer size ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "100000" ) ); } @Override public void close() { consumer.wakeup(); } @Override public void run() { try { log.info("Starting {} topic analysis", topicId); consumer.partitionsFor(topicId.topicName) .forEach(tp -> partitionStats.put(tp.partition(), new TopicAnalysisStats())); var seekOperations = SeekOperations.create(consumer, new ConsumerPosition(BEGINNING, topicId.topicName, null)); long summaryOffsetsRange = seekOperations.summaryOffsetsRange(); seekOperations.assignAndSeekNonEmptyPartitions(); while (!seekOperations.assignedPartitionsFullyPolled()) { var polled = consumer.pollEnhanced(Duration.ofSeconds(3)); polled.forEach(r -> { totalStats.apply(r); partitionStats.get(r.partition()).apply(r); }); updateProgress(seekOperations.offsetsProcessedFromSeek(), summaryOffsetsRange); } analysisTasksStore.setAnalysisResult(topicId, startedAt, totalStats, partitionStats); log.info("{} topic analysis finished", topicId); } catch (WakeupException | InterruptException cancelException) { log.info("{} topic analysis stopped", topicId); // calling cancel for cases when our thread was interrupted by some non-user cancellation reason analysisTasksStore.cancelAnalysis(topicId); } catch (Throwable th) { log.error("Error analyzing topic {}", topicId, th); analysisTasksStore.setAnalysisError(topicId, startedAt, th); } finally { consumer.close(); } } private void updateProgress(long processedOffsets, long summaryOffsetsRange) { if (processedOffsets > 0 && summaryOffsetsRange != 0) { analysisTasksStore.updateProgress( topicId, totalStats.totalMsgs, totalStats.keysSize.sum + totalStats.valuesSize.sum, Math.min(100.0, (((double) processedOffsets) / summaryOffsetsRange) * 100) ); } } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java ================================================ package com.provectus.kafka.ui.service.analyze; import com.provectus.kafka.ui.model.TopicAnalysisSizeStatsDTO; import com.provectus.kafka.ui.model.TopicAnalysisStatsDTO; import com.provectus.kafka.ui.model.TopicAnalysisStatsHourlyMsgCountsInnerDTO; import java.time.Duration; import java.time.Instant; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.datasketches.hll.HllSketch; import org.apache.datasketches.quantiles.DoublesSketch; import org.apache.datasketches.quantiles.UpdateDoublesSketch; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; class TopicAnalysisStats { Long totalMsgs = 0L; Long minOffset; Long maxOffset; Long minTimestamp; Long maxTimestamp; long nullKeys = 0L; long nullValues = 0L; final SizeStats keysSize = new SizeStats(); final SizeStats valuesSize = new SizeStats(); final HllSketch uniqKeys = new HllSketch(); final HllSketch uniqValues = new HllSketch(); final HourlyCounts hourlyCounts = new HourlyCounts(); static class SizeStats { long sum = 0; Long min; Long max; final UpdateDoublesSketch sizeSketch = DoublesSketch.builder().build(); void apply(int len) { sum += len; min = minNullable(min, len); max = maxNullable(max, len); sizeSketch.update(len); } TopicAnalysisSizeStatsDTO toDto() { return new TopicAnalysisSizeStatsDTO() .sum(sum) .min(min) .max(max) .avg((long) (((double) sum) / sizeSketch.getN())) .prctl50((long) sizeSketch.getQuantile(0.5)) .prctl75((long) sizeSketch.getQuantile(0.75)) .prctl95((long) sizeSketch.getQuantile(0.95)) .prctl99((long) sizeSketch.getQuantile(0.99)) .prctl999((long) sizeSketch.getQuantile(0.999)); } } static class HourlyCounts { // hour start ms -> count private final Map hourlyStats = new HashMap<>(); private final long minTs = Instant.now().minus(Duration.ofDays(14)).toEpochMilli(); void apply(ConsumerRecord rec) { if (rec.timestamp() > minTs) { var hourStart = rec.timestamp() - rec.timestamp() % (1_000 * 60 * 60); hourlyStats.compute(hourStart, (h, cnt) -> cnt == null ? 1 : cnt + 1); } } List toDto() { return hourlyStats.entrySet().stream() .sorted(Comparator.comparingLong(Map.Entry::getKey)) .map(e -> new TopicAnalysisStatsHourlyMsgCountsInnerDTO() .hourStart(e.getKey()) .count(e.getValue())) .collect(Collectors.toList()); } } void apply(ConsumerRecord rec) { totalMsgs++; minTimestamp = minNullable(minTimestamp, rec.timestamp()); maxTimestamp = maxNullable(maxTimestamp, rec.timestamp()); minOffset = minNullable(minOffset, rec.offset()); maxOffset = maxNullable(maxOffset, rec.offset()); hourlyCounts.apply(rec); if (rec.key() != null) { byte[] keyBytes = rec.key().get(); keysSize.apply(rec.serializedKeySize()); uniqKeys.update(keyBytes); } else { nullKeys++; } if (rec.value() != null) { byte[] valueBytes = rec.value().get(); valuesSize.apply(rec.serializedValueSize()); uniqValues.update(valueBytes); } else { nullValues++; } } TopicAnalysisStatsDTO toDto(@Nullable Integer partition) { return new TopicAnalysisStatsDTO() .partition(partition) .totalMsgs(totalMsgs) .minOffset(minOffset) .maxOffset(maxOffset) .minTimestamp(minTimestamp) .maxTimestamp(maxTimestamp) .nullKeys(nullKeys) .nullValues(nullValues) // because of hll error estimated size can be greater that actual msgs count .approxUniqKeys(Math.min(totalMsgs, (long) uniqKeys.getEstimate())) .approxUniqValues(Math.min(totalMsgs, (long) uniqValues.getEstimate())) .keySize(keysSize.toDto()) .valueSize(valuesSize.toDto()) .hourlyMsgCounts(hourlyCounts.toDto()); } private static Long maxNullable(@Nullable Long v1, long v2) { return v1 == null ? v2 : Math.max(v1, v2); } private static Long minNullable(@Nullable Long v1, long v2) { return v1 == null ? v2 : Math.min(v1, v2); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicIdentity.java ================================================ package com.provectus.kafka.ui.service.analyze; import com.provectus.kafka.ui.model.KafkaCluster; import lombok.EqualsAndHashCode; import lombok.ToString; @ToString @EqualsAndHashCode class TopicIdentity { final String clusterName; final String topicName; public TopicIdentity(KafkaCluster cluster, String topic) { this.clusterName = cluster.getName(); this.topicName = topic; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/audit/AuditRecord.java ================================================ package com.provectus.kafka.ui.service.audit; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.exception.CustomBaseException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.Resource; import com.provectus.kafka.ui.model.rbac.permission.PermissibleAction; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import lombok.SneakyThrows; import org.springframework.security.access.AccessDeniedException; record AuditRecord(String timestamp, String username, String clusterName, List resources, String operation, Object operationParams, OperationResult result) { static final JsonMapper MAPPER = new JsonMapper(); static { MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); } @SneakyThrows String toJson() { return MAPPER.writeValueAsString(this); } record AuditResource(String accessType, boolean alter, Resource type, @Nullable Object id) { private static AuditResource create(PermissibleAction action, Resource type, @Nullable Object id) { return new AuditResource(action.name(), action.isAlter(), type, id); } static List getAccessedResources(AccessContext ctx) { List resources = new ArrayList<>(); ctx.getClusterConfigActions() .forEach(a -> resources.add(create(a, Resource.CLUSTERCONFIG, null))); ctx.getTopicActions() .forEach(a -> resources.add(create(a, Resource.TOPIC, nameId(ctx.getTopic())))); ctx.getConsumerGroupActions() .forEach(a -> resources.add(create(a, Resource.CONSUMER, nameId(ctx.getConsumerGroup())))); ctx.getConnectActions() .forEach(a -> { Map resourceId = new LinkedHashMap<>(); resourceId.put("connect", ctx.getConnect()); if (ctx.getConnector() != null) { resourceId.put("connector", ctx.getConnector()); } resources.add(create(a, Resource.CONNECT, resourceId)); }); ctx.getSchemaActions() .forEach(a -> resources.add(create(a, Resource.SCHEMA, nameId(ctx.getSchema())))); ctx.getKsqlActions() .forEach(a -> resources.add(create(a, Resource.KSQL, null))); ctx.getAclActions() .forEach(a -> resources.add(create(a, Resource.ACL, null))); ctx.getAuditAction() .forEach(a -> resources.add(create(a, Resource.AUDIT, null))); return resources; } @Nullable private static Map nameId(@Nullable String name) { return name != null ? Map.of("name", name) : null; } } record OperationResult(boolean success, OperationError error) { static OperationResult successful() { return new OperationResult(true, null); } static OperationResult error(Throwable th) { OperationError err = OperationError.UNRECOGNIZED_ERROR; if (th instanceof AccessDeniedException) { err = OperationError.ACCESS_DENIED; } else if (th instanceof ValidationException) { err = OperationError.VALIDATION_ERROR; } else if (th instanceof CustomBaseException) { err = OperationError.EXECUTION_ERROR; } return new OperationResult(false, err); } enum OperationError { ACCESS_DENIED, VALIDATION_ERROR, EXECUTION_ERROR, UNRECOGNIZED_ERROR } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/audit/AuditService.java ================================================ package com.provectus.kafka.ui.service.audit; import static com.provectus.kafka.ui.config.ClustersProperties.AuditProperties.LogLevel.ALTER_ONLY; import static com.provectus.kafka.ui.service.MessagesService.createProducer; import com.google.common.annotations.VisibleForTesting; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.config.auth.AuthenticatedUser; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.AdminClientService; import com.provectus.kafka.ui.service.ClustersStorage; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.io.Closeable; import java.io.IOException; import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.userdetails.UserDetails; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; import reactor.core.publisher.Signal; @Slf4j @Service public class AuditService implements Closeable { private static final Mono NO_AUTH_USER = Mono.just(new AuthenticatedUser("Unknown", Set.of())); private static final Duration BLOCK_TIMEOUT = Duration.ofSeconds(5); private static final String DEFAULT_AUDIT_TOPIC_NAME = "__kui-audit-log"; private static final int DEFAULT_AUDIT_TOPIC_PARTITIONS = 1; private static final Map DEFAULT_AUDIT_TOPIC_CONFIG = Map.of( "retention.ms", String.valueOf(TimeUnit.DAYS.toMillis(7)), "cleanup.policy", "delete" ); private static final Map AUDIT_PRODUCER_CONFIG = Map.of( ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip" ); private static final Logger AUDIT_LOGGER = LoggerFactory.getLogger("audit"); private final Map auditWriters; @Autowired public AuditService(AdminClientService adminClientService, ClustersStorage clustersStorage) { Map auditWriters = new HashMap<>(); for (var cluster : clustersStorage.getKafkaClusters()) { Supplier adminClientSupplier = () -> adminClientService.get(cluster).block(BLOCK_TIMEOUT); createAuditWriter(cluster, adminClientSupplier, () -> createProducer(cluster, AUDIT_PRODUCER_CONFIG)) .ifPresent(writer -> auditWriters.put(cluster.getName(), writer)); } this.auditWriters = auditWriters; } @VisibleForTesting AuditService(Map auditWriters) { this.auditWriters = auditWriters; } @VisibleForTesting static Optional createAuditWriter(KafkaCluster cluster, Supplier acSupplier, Supplier> producerFactory) { var auditProps = cluster.getOriginalProperties().getAudit(); if (auditProps == null) { return Optional.empty(); } boolean topicAudit = Optional.ofNullable(auditProps.getTopicAuditEnabled()).orElse(false); boolean consoleAudit = Optional.ofNullable(auditProps.getConsoleAuditEnabled()).orElse(false); boolean alterLogOnly = Optional.ofNullable(auditProps.getLevel()).map(lvl -> lvl == ALTER_ONLY).orElse(true); if (!topicAudit && !consoleAudit) { return Optional.empty(); } if (!topicAudit) { log.info("Audit initialization finished for cluster '{}' (console only)", cluster.getName()); return Optional.of(consoleOnlyWriter(cluster, alterLogOnly)); } String auditTopicName = Optional.ofNullable(auditProps.getTopic()).orElse(DEFAULT_AUDIT_TOPIC_NAME); boolean topicAuditCanBeDone = createTopicIfNeeded(cluster, acSupplier, auditTopicName, auditProps); if (!topicAuditCanBeDone) { if (consoleAudit) { log.info( "Audit initialization finished for cluster '{}' (console only, topic audit init failed)", cluster.getName() ); return Optional.of(consoleOnlyWriter(cluster, alterLogOnly)); } return Optional.empty(); } log.info("Audit initialization finished for cluster '{}'", cluster.getName()); return Optional.of( new AuditWriter( cluster.getName(), alterLogOnly, auditTopicName, producerFactory.get(), consoleAudit ? AUDIT_LOGGER : null ) ); } private static AuditWriter consoleOnlyWriter(KafkaCluster cluster, boolean alterLogOnly) { return new AuditWriter(cluster.getName(), alterLogOnly, null, null, AUDIT_LOGGER); } /** * return true if topic created/existing and producing can be enabled. */ private static boolean createTopicIfNeeded(KafkaCluster cluster, Supplier acSupplier, String auditTopicName, ClustersProperties.AuditProperties auditProps) { ReactiveAdminClient ac; try { ac = acSupplier.get(); } catch (Exception e) { printAuditInitError(cluster, "Error while connecting to the cluster", e); return false; } boolean topicExists; try { topicExists = ac.listTopics(true).block(BLOCK_TIMEOUT).contains(auditTopicName); } catch (Exception e) { printAuditInitError(cluster, "Error checking audit topic existence", e); return false; } if (topicExists) { return true; } try { int topicPartitions = Optional.ofNullable(auditProps.getAuditTopicsPartitions()) .orElse(DEFAULT_AUDIT_TOPIC_PARTITIONS); Map topicConfig = new HashMap<>(DEFAULT_AUDIT_TOPIC_CONFIG); Optional.ofNullable(auditProps.getAuditTopicProperties()) .ifPresent(topicConfig::putAll); log.info("Creating audit topic '{}' for cluster '{}'", auditTopicName, cluster.getName()); ac.createTopic(auditTopicName, topicPartitions, null, topicConfig).block(BLOCK_TIMEOUT); log.info("Audit topic created for cluster '{}'", cluster.getName()); return true; } catch (Exception e) { printAuditInitError(cluster, "Error creating topic '%s'".formatted(auditTopicName), e); return false; } } private static void printAuditInitError(KafkaCluster cluster, String errorMsg, Exception cause) { log.error("-----------------------------------------------------------------"); log.error( "Error initializing Audit for cluster '{}'. Audit will be disabled. See error below: ", cluster.getName() ); log.error("{}", errorMsg, cause); log.error("-----------------------------------------------------------------"); } public boolean isAuditTopic(KafkaCluster cluster, String topic) { var writer = auditWriters.get(cluster.getName()); return writer != null && topic.equals(writer.targetTopic()) && writer.isTopicWritingEnabled(); } public void audit(AccessContext acxt, Signal sig) { if (sig.isOnComplete()) { extractUser(sig) .doOnNext(u -> sendAuditRecord(acxt, u)) .subscribe(); } else if (sig.isOnError()) { extractUser(sig) .doOnNext(u -> sendAuditRecord(acxt, u, sig.getThrowable())) .subscribe(); } } private Mono extractUser(Signal sig) { //see ReactiveSecurityContextHolder for impl details Object key = SecurityContext.class; if (sig.getContextView().hasKey(key)) { return sig.getContextView().>get(key) .map(context -> context.getAuthentication().getPrincipal()) .cast(UserDetails.class) .map(user -> { var roles = user.getAuthorities().stream().map(GrantedAuthority::getAuthority).collect(Collectors.toSet()); return new AuthenticatedUser(user.getUsername(), roles); }) .switchIfEmpty(NO_AUTH_USER); } else { return NO_AUTH_USER; } } private void sendAuditRecord(AccessContext ctx, AuthenticatedUser user) { sendAuditRecord(ctx, user, null); } private void sendAuditRecord(AccessContext ctx, AuthenticatedUser user, @Nullable Throwable th) { try { if (ctx.getCluster() != null) { var writer = auditWriters.get(ctx.getCluster()); if (writer != null) { writer.write(ctx, user, th); } } else { // cluster-independent operation AuditWriter.writeAppOperation(AUDIT_LOGGER, ctx, user, th); } } catch (Exception e) { log.warn("Error sending audit record", e); } } @Override public void close() throws IOException { auditWriters.values().forEach(AuditWriter::close); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/audit/AuditWriter.java ================================================ package com.provectus.kafka.ui.service.audit; import static java.nio.charset.StandardCharsets.UTF_8; import com.provectus.kafka.ui.config.auth.AuthenticatedUser; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.audit.AuditRecord.AuditResource; import com.provectus.kafka.ui.service.audit.AuditRecord.OperationResult; import java.io.Closeable; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.Optional; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.slf4j.Logger; @Slf4j record AuditWriter(String clusterName, boolean logAlterOperationsOnly, @Nullable String targetTopic, @Nullable KafkaProducer producer, @Nullable Logger consoleLogger) implements Closeable { boolean isTopicWritingEnabled() { return producer != null; } // application-level (cluster-independent) operation static void writeAppOperation(Logger consoleLogger, AccessContext ctx, AuthenticatedUser user, @Nullable Throwable th) { consoleLogger.info(createRecord(ctx, user, th).toJson()); } void write(AccessContext ctx, AuthenticatedUser user, @Nullable Throwable th) { write(createRecord(ctx, user, th)); } private void write(AuditRecord rec) { if (logAlterOperationsOnly && rec.resources().stream().noneMatch(AuditResource::alter)) { //we should only log alter operations, but this is read-only op return; } String json = rec.toJson(); if (consoleLogger != null) { consoleLogger.info(json); } if (targetTopic != null && producer != null) { producer.send( new ProducerRecord<>(targetTopic, null, json.getBytes(UTF_8)), (metadata, ex) -> { if (ex != null) { log.warn("Error sending Audit record to kafka for cluster {}", clusterName, ex); } }); } } private static AuditRecord createRecord(AccessContext ctx, AuthenticatedUser user, @Nullable Throwable th) { return new AuditRecord( DateTimeFormatter.ISO_INSTANT.format(Instant.now()), user.principal(), ctx.getCluster(), //can be null, if it is application-level action AuditResource.getAccessedResources(ctx), ctx.getOperationName(), ctx.getOperationParams(), th == null ? OperationResult.successful() : OperationResult.error(th) ); } @Override public void close() { Optional.ofNullable(producer).ifPresent(KafkaProducer::close); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/ConnectorInfo.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.provectus.kafka.ui.model.ConnectorTypeDTO; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.commons.collections4.CollectionUtils; import org.opendatadiscovery.oddrn.JdbcUrlParser; import org.opendatadiscovery.oddrn.model.HivePath; import org.opendatadiscovery.oddrn.model.MysqlPath; import org.opendatadiscovery.oddrn.model.PostgreSqlPath; import org.opendatadiscovery.oddrn.model.SnowflakePath; record ConnectorInfo(List inputs, List outputs) { static ConnectorInfo extract(String className, ConnectorTypeDTO type, Map config, List topicsFromApi, // can be empty for old Connect API versions Function topicOddrnBuilder) { return switch (className) { case "org.apache.kafka.connect.file.FileStreamSinkConnector", "org.apache.kafka.connect.file.FileStreamSourceConnector", "FileStreamSource", "FileStreamSink" -> extractFileIoConnector(type, topicsFromApi, config, topicOddrnBuilder); case "io.confluent.connect.s3.S3SinkConnector" -> extractS3Sink(type, topicsFromApi, config, topicOddrnBuilder); case "io.confluent.connect.jdbc.JdbcSinkConnector" -> extractJdbcSink(type, topicsFromApi, config, topicOddrnBuilder); case "io.debezium.connector.postgresql.PostgresConnector" -> extractDebeziumPg(config); case "io.debezium.connector.mysql.MySqlConnector" -> extractDebeziumMysql(config); default -> new ConnectorInfo( extractInputs(type, topicsFromApi, config, topicOddrnBuilder), extractOutputs(type, topicsFromApi, config, topicOddrnBuilder) ); }; } private static ConnectorInfo extractFileIoConnector(ConnectorTypeDTO type, List topics, Map config, Function topicOddrnBuilder) { return new ConnectorInfo( extractInputs(type, topics, config, topicOddrnBuilder), extractOutputs(type, topics, config, topicOddrnBuilder) ); } private static ConnectorInfo extractJdbcSink(ConnectorTypeDTO type, List topics, Map config, Function topicOddrnBuilder) { String tableNameFormat = (String) config.getOrDefault("table.name.format", "${topic}"); List targetTables = extractTopicNamesBestEffort(topics, config) .map(topic -> tableNameFormat.replace("${kafka}", topic)) .toList(); String connectionUrl = (String) config.get("connection.url"); List outputs = new ArrayList<>(); @Nullable var knownJdbcPath = new JdbcUrlParser().parse(connectionUrl); if (knownJdbcPath instanceof PostgreSqlPath p) { targetTables.forEach(t -> outputs.add(p.toBuilder().table(t).build().oddrn())); } if (knownJdbcPath instanceof MysqlPath p) { targetTables.forEach(t -> outputs.add(p.toBuilder().table(t).build().oddrn())); } if (knownJdbcPath instanceof HivePath p) { targetTables.forEach(t -> outputs.add(p.toBuilder().table(t).build().oddrn())); } if (knownJdbcPath instanceof SnowflakePath p) { targetTables.forEach(t -> outputs.add(p.toBuilder().table(t).build().oddrn())); } return new ConnectorInfo( extractInputs(type, topics, config, topicOddrnBuilder), outputs ); } private static ConnectorInfo extractDebeziumPg(Map config) { String host = (String) config.get("database.hostname"); String dbName = (String) config.get("database.dbname"); var inputs = List.of( PostgreSqlPath.builder() .host(host) .database(dbName) .build().oddrn() ); return new ConnectorInfo(inputs, List.of()); } private static ConnectorInfo extractDebeziumMysql(Map config) { String host = (String) config.get("database.hostname"); var inputs = List.of( MysqlPath.builder() .host(host) .build() .oddrn() ); return new ConnectorInfo(inputs, List.of()); } private static ConnectorInfo extractS3Sink(ConnectorTypeDTO type, List topics, Map config, Function topicOrrdnBuilder) { String bucketName = (String) config.get("s3.bucket.name"); String topicsDir = (String) config.getOrDefault("topics.dir", "topics"); String directoryDelim = (String) config.getOrDefault("directory.delim", "/"); List outputs = extractTopicNamesBestEffort(topics, config) .map(topic -> Oddrn.awsS3Oddrn(bucketName, topicsDir + directoryDelim + topic)) .toList(); return new ConnectorInfo( extractInputs(type, topics, config, topicOrrdnBuilder), outputs ); } private static List extractInputs(ConnectorTypeDTO type, List topicsFromApi, Map config, Function topicOrrdnBuilder) { return type == ConnectorTypeDTO.SINK ? extractTopicsOddrns(config, topicsFromApi, topicOrrdnBuilder) : List.of(); } private static List extractOutputs(ConnectorTypeDTO type, List topicsFromApi, Map config, Function topicOrrdnBuilder) { return type == ConnectorTypeDTO.SOURCE ? extractTopicsOddrns(config, topicsFromApi, topicOrrdnBuilder) : List.of(); } private static Stream extractTopicNamesBestEffort( // topic list can be empty for old Connect API versions List topicsFromApi, Map config ) { if (CollectionUtils.isNotEmpty(topicsFromApi)) { return topicsFromApi.stream(); } // trying to extract topic names from config String topicsString = (String) config.get("topics"); String topicString = (String) config.get("topic"); return Stream.of(topicsString, topicString) .filter(Objects::nonNull) .flatMap(str -> Stream.of(str.split(","))) .map(String::trim) .filter(s -> !s.isBlank()); } private static List extractTopicsOddrns(Map config, List topicsFromApi, Function topicOrrdnBuilder) { return extractTopicNamesBestEffort(topicsFromApi, config) .map(topicOrrdnBuilder) .toList(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/ConnectorsExporter.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.provectus.kafka.ui.connect.model.ConnectorTopics; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.KafkaConnectService; import java.net.URI; import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; import org.opendatadiscovery.client.model.DataEntity; import org.opendatadiscovery.client.model.DataEntityList; import org.opendatadiscovery.client.model.DataEntityType; import org.opendatadiscovery.client.model.DataSource; import org.opendatadiscovery.client.model.DataTransformer; import org.opendatadiscovery.client.model.MetadataExtension; import reactor.core.publisher.Flux; @RequiredArgsConstructor class ConnectorsExporter { private final KafkaConnectService kafkaConnectService; Flux export(KafkaCluster cluster) { return kafkaConnectService.getConnects(cluster) .flatMap(connect -> kafkaConnectService.getConnectorNamesWithErrorsSuppress(cluster, connect.getName()) .flatMap(connectorName -> kafkaConnectService.getConnector(cluster, connect.getName(), connectorName)) .flatMap(connectorDTO -> kafkaConnectService.getConnectorTopics(cluster, connect.getName(), connectorDTO.getName()) .map(topics -> createConnectorDataEntity(cluster, connect, connectorDTO, topics))) .buffer(100) .map(connectDataEntities -> { String dsOddrn = Oddrn.connectDataSourceOddrn(connect.getAddress()); return new DataEntityList() .dataSourceOddrn(dsOddrn) .items(connectDataEntities); }) ); } Flux getConnectDataSources(KafkaCluster cluster) { return kafkaConnectService.getConnects(cluster) .map(ConnectorsExporter::toDataSource); } private static DataSource toDataSource(ConnectDTO connect) { return new DataSource() .oddrn(Oddrn.connectDataSourceOddrn(connect.getAddress())) .name(connect.getName()) .description("Kafka Connect"); } private static DataEntity createConnectorDataEntity(KafkaCluster cluster, ConnectDTO connect, ConnectorDTO connector, ConnectorTopics connectorTopics) { var metadata = new HashMap<>(extractMetadata(connector)); metadata.put("type", connector.getType().name()); var info = extractConnectorInfo(cluster, connector, connectorTopics); DataTransformer transformer = new DataTransformer(); transformer.setInputs(info.inputs()); transformer.setOutputs(info.outputs()); return new DataEntity() .oddrn(Oddrn.connectorOddrn(connect.getAddress(), connector.getName())) .name(connector.getName()) .description("Kafka Connector \"%s\" (%s)".formatted(connector.getName(), connector.getType())) .type(DataEntityType.JOB) .dataTransformer(transformer) .metadata(List.of( new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) .metadata(metadata))); } private static Map extractMetadata(ConnectorDTO connector) { // will be sanitized by KafkaConfigSanitizer (if it's enabled) return connector.getConfig(); } private static ConnectorInfo extractConnectorInfo(KafkaCluster cluster, ConnectorDTO connector, ConnectorTopics topics) { return ConnectorInfo.extract( (String) connector.getConfig().get("connector.class"), connector.getType(), connector.getConfig(), topics.getTopics(), topic -> Oddrn.topicOddrn(cluster, topic) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/OddExporter.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.KafkaConnectService; import com.provectus.kafka.ui.service.StatisticsCache; import java.util.function.Predicate; import java.util.regex.Pattern; import org.opendatadiscovery.client.ApiClient; import org.opendatadiscovery.client.api.OpenDataDiscoveryIngestionApi; import org.opendatadiscovery.client.model.DataEntityList; import org.opendatadiscovery.client.model.DataSource; import org.opendatadiscovery.client.model.DataSourceList; import org.springframework.http.HttpHeaders; import reactor.core.publisher.Mono; class OddExporter { private final OpenDataDiscoveryIngestionApi oddApi; private final TopicsExporter topicsExporter; private final ConnectorsExporter connectorsExporter; public OddExporter(StatisticsCache statisticsCache, KafkaConnectService connectService, OddIntegrationProperties oddIntegrationProperties) { this( createApiClient(oddIntegrationProperties), new TopicsExporter(createTopicsFilter(oddIntegrationProperties), statisticsCache), new ConnectorsExporter(connectService) ); } @VisibleForTesting OddExporter(OpenDataDiscoveryIngestionApi oddApi, TopicsExporter topicsExporter, ConnectorsExporter connectorsExporter) { this.oddApi = oddApi; this.topicsExporter = topicsExporter; this.connectorsExporter = connectorsExporter; } private static Predicate createTopicsFilter(OddIntegrationProperties properties) { if (properties.getTopicsRegex() == null) { return topic -> !topic.startsWith("_"); } Pattern pattern = Pattern.compile(properties.getTopicsRegex()); return topic -> pattern.matcher(topic).matches(); } private static OpenDataDiscoveryIngestionApi createApiClient(OddIntegrationProperties properties) { Preconditions.checkNotNull(properties.getUrl(), "ODD url not set"); Preconditions.checkNotNull(properties.getToken(), "ODD token not set"); var apiClient = new ApiClient() .setBasePath(properties.getUrl()) .addDefaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + properties.getToken()); return new OpenDataDiscoveryIngestionApi(apiClient); } public Mono export(KafkaCluster cluster) { return exportTopics(cluster) .then(exportKafkaConnects(cluster)); } private Mono exportTopics(KafkaCluster c) { return createKafkaDataSource(c) .thenMany(topicsExporter.export(c)) .concatMap(this::sendDataEntities) .then(); } private Mono exportKafkaConnects(KafkaCluster cluster) { return createConnectDataSources(cluster) .thenMany(connectorsExporter.export(cluster)) .concatMap(this::sendDataEntities) .then(); } private Mono createConnectDataSources(KafkaCluster cluster) { return connectorsExporter.getConnectDataSources(cluster) .buffer(100) .concatMap(dataSources -> oddApi.createDataSource(new DataSourceList().items(dataSources))) .then(); } private Mono createKafkaDataSource(KafkaCluster cluster) { String clusterOddrn = Oddrn.clusterOddrn(cluster); return oddApi.createDataSource( new DataSourceList() .addItemsItem( new DataSource() .oddrn(clusterOddrn) .name(cluster.getName()) .description("Kafka cluster") ) ); } private Mono sendDataEntities(DataEntityList dataEntityList) { return oddApi.postDataEntityList(dataEntityList); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/OddExporterScheduler.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.provectus.kafka.ui.service.ClustersStorage; import lombok.RequiredArgsConstructor; import org.springframework.scheduling.annotation.Scheduled; import reactor.core.publisher.Flux; import reactor.core.scheduler.Schedulers; @RequiredArgsConstructor class OddExporterScheduler { private final ClustersStorage clustersStorage; private final OddExporter oddExporter; @Scheduled(fixedRateString = "${kafka.send-stats-to-odd-millis:30000}") public void sendMetricsToOdd() { Flux.fromIterable(clustersStorage.getKafkaClusters()) .parallel() .runOn(Schedulers.parallel()) .flatMap(oddExporter::export) .then() .block(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/OddIntegrationConfig.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.provectus.kafka.ui.service.ClustersStorage; import com.provectus.kafka.ui.service.KafkaConnectService; import com.provectus.kafka.ui.service.StatisticsCache; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration @ConditionalOnProperty(value = "integration.odd.url") class OddIntegrationConfig { @Bean OddIntegrationProperties oddIntegrationProperties() { return new OddIntegrationProperties(); } @Bean OddExporter oddExporter(StatisticsCache statisticsCache, KafkaConnectService connectService, OddIntegrationProperties oddIntegrationProperties) { return new OddExporter(statisticsCache, connectService, oddIntegrationProperties); } @Bean OddExporterScheduler oddExporterScheduler(ClustersStorage storage, OddExporter exporter) { return new OddExporterScheduler(storage, exporter); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/OddIntegrationProperties.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; @Data @ConfigurationProperties("integration.odd") public class OddIntegrationProperties { String url; String token; String topicsRegex; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/Oddrn.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.provectus.kafka.ui.model.KafkaCluster; import java.net.URI; import java.util.stream.Collectors; import java.util.stream.Stream; import org.opendatadiscovery.oddrn.model.AwsS3Path; import org.opendatadiscovery.oddrn.model.KafkaConnectorPath; import org.opendatadiscovery.oddrn.model.KafkaPath; public final class Oddrn { private Oddrn() { } static String clusterOddrn(KafkaCluster cluster) { return KafkaPath.builder() .cluster(bootstrapServersForOddrn(cluster.getBootstrapServers())) .build() .oddrn(); } static KafkaPath topicOddrnPath(KafkaCluster cluster, String topic) { return KafkaPath.builder() .cluster(bootstrapServersForOddrn(cluster.getBootstrapServers())) .topic(topic) .build(); } static String topicOddrn(KafkaCluster cluster, String topic) { return topicOddrnPath(cluster, topic).oddrn(); } static String awsS3Oddrn(String bucket, String key) { return AwsS3Path.builder() .bucket(bucket) .key(key) .build() .oddrn(); } static String connectDataSourceOddrn(String connectUrl) { return KafkaConnectorPath.builder() .host(normalizedConnectHosts(connectUrl)) .build() .oddrn(); } private static String normalizedConnectHosts(String connectUrlStr) { return Stream.of(connectUrlStr.split(",")) .map(String::trim) .sorted() .map(url -> { var uri = URI.create(url); String host = uri.getHost(); String portSuffix = (uri.getPort() > 0 ? (":" + uri.getPort()) : ""); return host + portSuffix; }) .collect(Collectors.joining(",")); } static String connectorOddrn(String connectUrl, String connectorName) { return KafkaConnectorPath.builder() .host(normalizedConnectHosts(connectUrl)) .connector(connectorName) .build() .oddrn(); } private static String bootstrapServersForOddrn(String bootstrapServers) { return Stream.of(bootstrapServers.split(",")) .map(String::trim) .sorted() .collect(Collectors.joining(",")); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/SchemaReferencesResolver.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.sr.model.SchemaReference; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; import reactor.core.publisher.Mono; // logic copied from AbstractSchemaProvider:resolveReferences // https://github.com/confluentinc/schema-registry/blob/fd59613e2c5adf62e36705307f420712e4c8c1ea/client/src/main/java/io/confluent/kafka/schemaregistry/AbstractSchemaProvider.java#L54 class SchemaReferencesResolver { private final KafkaSrClientApi client; SchemaReferencesResolver(KafkaSrClientApi client) { this.client = client; } Mono> resolve(List refs) { return resolveReferences(refs, new Resolving(ImmutableMap.of(), ImmutableSet.of())) .map(Resolving::resolved); } private record Resolving(ImmutableMap resolved, ImmutableSet visited) { Resolving visit(String name) { return new Resolving(resolved, ImmutableSet.builder().addAll(visited).add(name).build()); } Resolving resolve(String ref, String schema) { return new Resolving(ImmutableMap.builder().putAll(resolved).put(ref, schema).build(), visited); } } private Mono resolveReferences(@Nullable List refs, Resolving initState) { Mono result = Mono.just(initState); for (SchemaReference reference : Optional.ofNullable(refs).orElse(List.of())) { result = result.flatMap(state -> { if (state.visited().contains(reference.getName())) { return Mono.just(state); } else { final var newState = state.visit(reference.getName()); return client.getSubjectVersion(reference.getSubject(), String.valueOf(reference.getVersion()), true) .flatMap(subj -> resolveReferences(subj.getReferences(), newState) .map(withNewRefs -> withNewRefs.resolve(reference.getName(), subj.getSchema()))); } }); } return result; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporter.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import com.google.common.collect.ImmutableMap; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.service.StatisticsCache; import com.provectus.kafka.ui.service.integration.odd.schema.DataSetFieldsExtractors; import com.provectus.kafka.ui.sr.model.SchemaSubject; import java.net.URI; import java.util.List; import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.opendatadiscovery.client.model.DataEntity; import org.opendatadiscovery.client.model.DataEntityList; import org.opendatadiscovery.client.model.DataEntityType; import org.opendatadiscovery.client.model.DataSet; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.MetadataExtension; import org.opendatadiscovery.oddrn.model.KafkaPath; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Slf4j @RequiredArgsConstructor class TopicsExporter { private final Predicate topicFilter; private final StatisticsCache statisticsCache; Flux export(KafkaCluster cluster) { String clusterOddrn = Oddrn.clusterOddrn(cluster); Statistics stats = statisticsCache.get(cluster); return Flux.fromIterable(stats.getTopicDescriptions().keySet()) .filter(topicFilter) .flatMap(topic -> createTopicDataEntity(cluster, topic, stats)) .onErrorContinue( (th, topic) -> log.warn("Error exporting data for topic {}, cluster {}", topic, cluster.getName(), th)) .buffer(100) .map(topicsEntities -> new DataEntityList() .dataSourceOddrn(clusterOddrn) .items(topicsEntities)); } private Mono createTopicDataEntity(KafkaCluster cluster, String topic, Statistics stats) { KafkaPath topicOddrnPath = Oddrn.topicOddrnPath(cluster, topic); return Mono.zip( getTopicSchema(cluster, topic, topicOddrnPath, true), getTopicSchema(cluster, topic, topicOddrnPath, false) ) .map(keyValueFields -> { var dataset = new DataSet(); keyValueFields.getT1().forEach(dataset::addFieldListItem); keyValueFields.getT2().forEach(dataset::addFieldListItem); return new DataEntity() .name(topic) .description("Kafka topic \"%s\"".formatted(topic)) .oddrn(Oddrn.topicOddrn(cluster, topic)) .type(DataEntityType.KAFKA_TOPIC) .dataset(dataset) .addMetadataItem( new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) .metadata(getTopicMetadata(topic, stats))); } ); } private Map getNonDefaultConfigs(String topic, Statistics stats) { List config = stats.getTopicConfigs().get(topic); if (config == null) { return Map.of(); } return config.stream() .filter(c -> c.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG) .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); } private Map getTopicMetadata(String topic, Statistics stats) { TopicDescription topicDescription = stats.getTopicDescriptions().get(topic); return ImmutableMap.builder() .put("partitions", topicDescription.partitions().size()) .put("replication_factor", topicDescription.partitions().get(0).replicas().size()) .putAll(getNonDefaultConfigs(topic, stats)) .build(); } //returns empty list if schemaRegistry is not configured or assumed subject not found private Mono> getTopicSchema(KafkaCluster cluster, String topic, KafkaPath topicOddrn, boolean isKey) { if (cluster.getSchemaRegistryClient() == null) { return Mono.just(List.of()); } String subject = topic + (isKey ? "-key" : "-value"); return getSubjWithResolvedRefs(cluster, subject) .map(t -> DataSetFieldsExtractors.extract(t.getT1(), t.getT2(), topicOddrn, isKey)) .onErrorResume(WebClientResponseException.NotFound.class, th -> Mono.just(List.of())) .onErrorMap(WebClientResponseException.class, err -> new IllegalStateException("Error retrieving subject %s".formatted(subject), err)); } private Mono>> getSubjWithResolvedRefs(KafkaCluster cluster, String subjectName) { return cluster.getSchemaRegistryClient() .mono(client -> client.getSubjectVersion(subjectName, "latest", false) .flatMap(subj -> new SchemaReferencesResolver(client).resolve(subj.getReferences()) .map(resolvedRefs -> Tuples.of(subj, resolvedRefs)))); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/schema/AvroExtractor.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import com.google.common.collect.ImmutableSet; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import java.util.ArrayList; import java.util.List; import org.apache.avro.Schema; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.oddrn.model.KafkaPath; final class AvroExtractor { private AvroExtractor() { } static List extract(AvroSchema avroSchema, KafkaPath topicOddrn, boolean isKey) { var schema = avroSchema.rawSchema(); List result = new ArrayList<>(); result.add(DataSetFieldsExtractors.rootField(topicOddrn, isKey)); extract( schema, topicOddrn.oddrn() + "/columns/" + (isKey ? "key" : "value"), null, null, null, false, ImmutableSet.of(), result ); return result; } private static void extract(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, Boolean nullable, ImmutableSet registeredRecords, List sink ) { switch (schema.getType()) { case RECORD -> extractRecord(schema, parentOddr, oddrn, name, doc, nullable, registeredRecords, sink); case UNION -> extractUnion(schema, parentOddr, oddrn, name, doc, registeredRecords, sink); case ARRAY -> extractArray(schema, parentOddr, oddrn, name, doc, nullable, registeredRecords, sink); case MAP -> extractMap(schema, parentOddr, oddrn, name, doc, nullable, registeredRecords, sink); default -> extractPrimitive(schema, parentOddr, oddrn, name, doc, nullable, sink); } } private static DataSetField createDataSetField(String name, String doc, String parentOddrn, String oddrn, Schema schema, Boolean nullable) { return new DataSetField() .name(name) .description(doc) .parentFieldOddrn(parentOddrn) .oddrn(oddrn) .type(mapSchema(schema, nullable)); } private static void extractRecord(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, Boolean nullable, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; if (!isRoot) { sink.add(createDataSetField(name, doc, parentOddr, oddrn, schema, nullable)); if (registeredRecords.contains(schema.getFullName())) { // avoiding recursion by checking if record already registered in parsing chain return; } } var newRegisteredRecords = ImmutableSet.builder() .addAll(registeredRecords) .add(schema.getFullName()) .build(); schema.getFields().forEach(f -> extract( f.schema(), isRoot ? parentOddr : oddrn, isRoot ? parentOddr + "/" + f.name() : oddrn + "/fields/" + f.name(), f.name(), f.doc(), false, newRegisteredRecords, sink )); } private static void extractUnion(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; boolean containsNull = schema.getTypes().stream().map(Schema::getType).anyMatch(t -> t == Schema.Type.NULL); // if it is not root and there is only 2 values for union (null and smth else) // we registering this field as optional without mentioning union if (!isRoot && containsNull && schema.getTypes().size() == 2) { var nonNullSchema = schema.getTypes().stream() .filter(s -> s.getType() != Schema.Type.NULL) .findFirst() .orElseThrow(IllegalStateException::new); extract( nonNullSchema, parentOddr, oddrn, name, doc, true, registeredRecords, sink ); return; } oddrn = isRoot ? parentOddr + "/union" : oddrn; if (isRoot) { sink.add(createDataSetField("Avro root union", doc, parentOddr, oddrn, schema, containsNull)); } else { sink.add(createDataSetField(name, doc, parentOddr, oddrn, schema, containsNull)); } for (Schema t : schema.getTypes()) { if (t.getType() != Schema.Type.NULL) { extract( t, oddrn, oddrn + "/values/" + t.getName(), t.getName(), t.getDoc(), containsNull, registeredRecords, sink ); } } } private static void extractArray(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, Boolean nullable, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; oddrn = isRoot ? parentOddr + "/array" : oddrn; if (isRoot) { sink.add(createDataSetField("Avro root Array", doc, parentOddr, oddrn, schema, nullable)); } else { sink.add(createDataSetField(name, doc, parentOddr, oddrn, schema, nullable)); } extract( schema.getElementType(), oddrn, oddrn + "/items/" + schema.getElementType().getName(), schema.getElementType().getName(), schema.getElementType().getDoc(), false, registeredRecords, sink ); } private static void extractMap(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, Boolean nullable, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; oddrn = isRoot ? parentOddr + "/map" : oddrn; if (isRoot) { sink.add(createDataSetField("Avro root map", doc, parentOddr, oddrn, schema, nullable)); } else { sink.add(createDataSetField(name, doc, parentOddr, oddrn, schema, nullable)); } extract( new Schema.Parser().parse("\"string\""), oddrn, oddrn + "/key", "key", null, nullable, registeredRecords, sink ); extract( schema.getValueType(), oddrn, oddrn + "/value", "value", null, nullable, registeredRecords, sink ); } private static void extractPrimitive(Schema schema, String parentOddr, String oddrn, //null for root String name, String doc, Boolean nullable, List sink) { boolean isRoot = oddrn == null; String primOddrn = isRoot ? (parentOddr + "/" + schema.getType()) : oddrn; if (isRoot) { sink.add(createDataSetField("Root avro " + schema.getType(), doc, parentOddr, primOddrn, schema, nullable)); } else { sink.add(createDataSetField(name, doc, parentOddr, primOddrn, schema, nullable)); } } private static DataSetFieldType.TypeEnum mapType(Schema.Type type) { return switch (type) { case INT, LONG -> DataSetFieldType.TypeEnum.INTEGER; case FLOAT, DOUBLE, FIXED -> DataSetFieldType.TypeEnum.NUMBER; case STRING, ENUM -> DataSetFieldType.TypeEnum.STRING; case BOOLEAN -> DataSetFieldType.TypeEnum.BOOLEAN; case BYTES -> DataSetFieldType.TypeEnum.BINARY; case ARRAY -> DataSetFieldType.TypeEnum.LIST; case RECORD -> DataSetFieldType.TypeEnum.STRUCT; case MAP -> DataSetFieldType.TypeEnum.MAP; case UNION -> DataSetFieldType.TypeEnum.UNION; case NULL -> DataSetFieldType.TypeEnum.UNKNOWN; }; } private static DataSetFieldType mapSchema(Schema schema, Boolean nullable) { return new DataSetFieldType() .logicalType(logicalType(schema)) .isNullable(nullable) .type(mapType(schema.getType())); } private static String logicalType(Schema schema) { return schema.getType() == Schema.Type.RECORD ? schema.getFullName() : schema.getType().toString().toLowerCase(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/schema/DataSetFieldsExtractors.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import com.provectus.kafka.ui.sr.model.SchemaSubject; import com.provectus.kafka.ui.sr.model.SchemaType; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.json.JsonSchema; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import java.util.List; import java.util.Map; import java.util.Optional; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.oddrn.model.KafkaPath; public final class DataSetFieldsExtractors { public static List extract(SchemaSubject subject, Map resolvedRefs, KafkaPath topicOddrn, boolean isKey) { SchemaType schemaType = Optional.ofNullable(subject.getSchemaType()).orElse(SchemaType.AVRO); return switch (schemaType) { case AVRO -> AvroExtractor.extract( new AvroSchema(subject.getSchema(), List.of(), resolvedRefs, null), topicOddrn, isKey); case JSON -> JsonSchemaExtractor.extract( new JsonSchema(subject.getSchema(), List.of(), resolvedRefs, null), topicOddrn, isKey); case PROTOBUF -> ProtoExtractor.extract( new ProtobufSchema(subject.getSchema(), List.of(), resolvedRefs, null, null), topicOddrn, isKey); }; } static DataSetField rootField(KafkaPath topicOddrn, boolean isKey) { var rootOddrn = topicOddrn.oddrn() + "/columns/" + (isKey ? "key" : "value"); return new DataSetField() .name(isKey ? "key" : "value") .description("Topic's " + (isKey ? "key" : "value") + " schema") .parentFieldOddrn(topicOddrn.oddrn()) .oddrn(rootOddrn) .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .isNullable(true)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/schema/JsonSchemaExtractor.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import com.google.common.collect.ImmutableSet; import com.provectus.kafka.ui.sr.model.SchemaSubject; import io.confluent.kafka.schemaregistry.json.JsonSchema; import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import javax.annotation.Nullable; import org.everit.json.schema.ArraySchema; import org.everit.json.schema.BooleanSchema; import org.everit.json.schema.CombinedSchema; import org.everit.json.schema.FalseSchema; import org.everit.json.schema.NullSchema; import org.everit.json.schema.NumberSchema; import org.everit.json.schema.ObjectSchema; import org.everit.json.schema.ReferenceSchema; import org.everit.json.schema.Schema; import org.everit.json.schema.StringSchema; import org.everit.json.schema.TrueSchema; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.client.model.MetadataExtension; import org.opendatadiscovery.oddrn.model.KafkaPath; final class JsonSchemaExtractor { private JsonSchemaExtractor() { } static List extract(JsonSchema jsonSchema, KafkaPath topicOddrn, boolean isKey) { Schema schema = jsonSchema.rawSchema(); List result = new ArrayList<>(); result.add(DataSetFieldsExtractors.rootField(topicOddrn, isKey)); extract( schema, topicOddrn.oddrn() + "/columns/" + (isKey ? "key" : "value"), null, null, null, ImmutableSet.of(), result ); return result; } private static void extract(Schema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, ImmutableSet registeredRecords, List sink) { if (schema instanceof ReferenceSchema s) { Optional.ofNullable(s.getReferredSchema()) .ifPresent(refSchema -> extract(refSchema, parentOddr, oddrn, name, nullable, registeredRecords, sink)); } else if (schema instanceof ObjectSchema s) { extractObject(s, parentOddr, oddrn, name, nullable, registeredRecords, sink); } else if (schema instanceof ArraySchema s) { extractArray(s, parentOddr, oddrn, name, nullable, registeredRecords, sink); } else if (schema instanceof CombinedSchema cs) { extractCombined(cs, parentOddr, oddrn, name, nullable, registeredRecords, sink); } else if (schema instanceof BooleanSchema || schema instanceof NumberSchema || schema instanceof StringSchema || schema instanceof NullSchema ) { extractPrimitive(schema, parentOddr, oddrn, name, nullable, sink); } else { extractUnknown(schema, parentOddr, oddrn, name, nullable, sink); } } private static void extractPrimitive(Schema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, List sink) { boolean isRoot = oddrn == null; sink.add( createDataSetField( schema, isRoot ? "Root JSON primitive" : name, parentOddr, isRoot ? (parentOddr + "/" + logicalTypeName(schema)) : oddrn, mapType(schema), logicalTypeName(schema), nullable ) ); } private static void extractUnknown(Schema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, List sink) { boolean isRoot = oddrn == null; sink.add( createDataSetField( schema, isRoot ? "Root type " + logicalTypeName(schema) : name, parentOddr, isRoot ? (parentOddr + "/" + logicalTypeName(schema)) : oddrn, DataSetFieldType.TypeEnum.UNKNOWN, logicalTypeName(schema), nullable ) ); } private static void extractObject(ObjectSchema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; // schemaLocation can be null for empty object schemas (like if it used in anyOf) @Nullable var schemaLocation = schema.getSchemaLocation(); if (!isRoot) { sink.add(createDataSetField( schema, name, parentOddr, oddrn, DataSetFieldType.TypeEnum.STRUCT, logicalTypeName(schema), nullable )); if (schemaLocation != null && registeredRecords.contains(schemaLocation)) { // avoiding recursion by checking if record already registered in parsing chain return; } } var newRegisteredRecords = schemaLocation == null ? registeredRecords : ImmutableSet.builder() .addAll(registeredRecords) .add(schemaLocation) .build(); schema.getPropertySchemas().forEach((propertyName, propertySchema) -> { boolean required = schema.getRequiredProperties().contains(propertyName); extract( propertySchema, isRoot ? parentOddr : oddrn, isRoot ? parentOddr + "/" + propertyName : oddrn + "/fields/" + propertyName, propertyName, !required, newRegisteredRecords, sink ); }); } private static void extractArray(ArraySchema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, ImmutableSet registeredRecords, List sink) { boolean isRoot = oddrn == null; oddrn = isRoot ? parentOddr + "/array" : oddrn; if (isRoot) { sink.add( createDataSetField( schema, "Json array root", parentOddr, oddrn, DataSetFieldType.TypeEnum.LIST, "array", nullable )); } else { sink.add( createDataSetField( schema, name, parentOddr, oddrn, DataSetFieldType.TypeEnum.LIST, "array", nullable )); } @Nullable var itemsSchema = schema.getAllItemSchema(); if (itemsSchema != null) { extract( itemsSchema, oddrn, oddrn + "/items/" + logicalTypeName(itemsSchema), logicalTypeName(itemsSchema), false, registeredRecords, sink ); } } private static void extractCombined(CombinedSchema schema, String parentOddr, String oddrn, //null for root String name, Boolean nullable, ImmutableSet registeredRecords, List sink) { String combineType = "unknown"; if (schema.getCriterion() == CombinedSchema.ALL_CRITERION) { combineType = "allOf"; } if (schema.getCriterion() == CombinedSchema.ANY_CRITERION) { combineType = "anyOf"; } if (schema.getCriterion() == CombinedSchema.ONE_CRITERION) { combineType = "oneOf"; } boolean isRoot = oddrn == null; oddrn = isRoot ? (parentOddr + "/" + combineType) : (oddrn + "/" + combineType); sink.add( createDataSetField( schema, isRoot ? "Root %s".formatted(combineType) : name, parentOddr, oddrn, DataSetFieldType.TypeEnum.UNION, combineType, nullable ).addMetadataItem(new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) .metadata(Map.of("criterion", combineType))) ); for (Schema subschema : schema.getSubschemas()) { extract( subschema, oddrn, oddrn + "/values/" + logicalTypeName(subschema), logicalTypeName(subschema), nullable, registeredRecords, sink ); } } private static String getDescription(Schema schema) { return Optional.ofNullable(schema.getTitle()) .orElse(schema.getDescription()); } private static String logicalTypeName(Schema schema) { return schema.getClass() .getSimpleName() .replace("Schema", ""); } private static DataSetField createDataSetField(Schema schema, String name, String parentOddrn, String oddrn, DataSetFieldType.TypeEnum type, String logicalType, Boolean nullable) { return new DataSetField() .name(name) .parentFieldOddrn(parentOddrn) .oddrn(oddrn) .description(getDescription(schema)) .type( new DataSetFieldType() .isNullable(nullable) .logicalType(logicalType) .type(type) ); } private static DataSetFieldType.TypeEnum mapType(Schema type) { if (type instanceof NumberSchema) { return DataSetFieldType.TypeEnum.NUMBER; } if (type instanceof StringSchema) { return DataSetFieldType.TypeEnum.STRING; } if (type instanceof BooleanSchema || type instanceof TrueSchema || type instanceof FalseSchema) { return DataSetFieldType.TypeEnum.BOOLEAN; } if (type instanceof ObjectSchema) { return DataSetFieldType.TypeEnum.STRUCT; } if (type instanceof ReferenceSchema s) { return mapType(s.getReferredSchema()); } if (type instanceof CombinedSchema) { return DataSetFieldType.TypeEnum.UNION; } return DataSetFieldType.TypeEnum.UNKNOWN; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/integration/odd/schema/ProtoExtractor.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import com.google.common.collect.ImmutableSet; import com.google.protobuf.BoolValue; import com.google.protobuf.BytesValue; import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.DoubleValue; import com.google.protobuf.Duration; import com.google.protobuf.FloatValue; import com.google.protobuf.Int32Value; import com.google.protobuf.Int64Value; import com.google.protobuf.StringValue; import com.google.protobuf.Timestamp; import com.google.protobuf.UInt32Value; import com.google.protobuf.UInt64Value; import com.google.protobuf.Value; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import java.util.ArrayList; import java.util.List; import java.util.Set; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.client.model.DataSetFieldType.TypeEnum; import org.opendatadiscovery.oddrn.model.KafkaPath; final class ProtoExtractor { private static final Set PRIMITIVES_WRAPPER_TYPE_NAMES = Set.of( BoolValue.getDescriptor().getFullName(), Int32Value.getDescriptor().getFullName(), UInt32Value.getDescriptor().getFullName(), Int64Value.getDescriptor().getFullName(), UInt64Value.getDescriptor().getFullName(), StringValue.getDescriptor().getFullName(), BytesValue.getDescriptor().getFullName(), FloatValue.getDescriptor().getFullName(), DoubleValue.getDescriptor().getFullName() ); private ProtoExtractor() { } static List extract(ProtobufSchema protobufSchema, KafkaPath topicOddrn, boolean isKey) { Descriptor schema = protobufSchema.toDescriptor(); List result = new ArrayList<>(); result.add(DataSetFieldsExtractors.rootField(topicOddrn, isKey)); var rootOddrn = topicOddrn.oddrn() + "/columns/" + (isKey ? "key" : "value"); schema.getFields().forEach(f -> extract(f, rootOddrn, rootOddrn + "/" + f.getName(), f.getName(), !f.isRequired(), f.isRepeated(), ImmutableSet.of(schema.getFullName()), result )); return result; } private static void extract(Descriptors.FieldDescriptor field, String parentOddr, String oddrn, //null for root String name, boolean nullable, boolean repeated, ImmutableSet registeredRecords, List sink) { if (repeated) { extractRepeated(field, parentOddr, oddrn, name, nullable, registeredRecords, sink); } else if (field.getType() == Descriptors.FieldDescriptor.Type.MESSAGE) { extractMessage(field, parentOddr, oddrn, name, nullable, registeredRecords, sink); } else { extractPrimitive(field, parentOddr, oddrn, name, nullable, sink); } } // converts some(!) Protobuf Well-known type (from google.protobuf.* packages) // see JsonFormat::buildWellKnownTypePrinters for impl details private static boolean extractProtoWellKnownType(Descriptors.FieldDescriptor field, String parentOddr, String oddrn, //null for root String name, boolean nullable, List sink) { // all well-known types are messages if (field.getType() != Descriptors.FieldDescriptor.Type.MESSAGE) { return false; } String typeName = field.getMessageType().getFullName(); if (typeName.equals(Timestamp.getDescriptor().getFullName())) { sink.add(createDataSetField(name, parentOddr, oddrn, TypeEnum.DATETIME, typeName, nullable)); return true; } if (typeName.equals(Duration.getDescriptor().getFullName())) { sink.add(createDataSetField(name, parentOddr, oddrn, TypeEnum.DURATION, typeName, nullable)); return true; } if (typeName.equals(Value.getDescriptor().getFullName())) { //TODO: use ANY type when it will appear in ODD sink.add(createDataSetField(name, parentOddr, oddrn, TypeEnum.UNKNOWN, typeName, nullable)); return true; } if (PRIMITIVES_WRAPPER_TYPE_NAMES.contains(typeName)) { var wrapped = field.getMessageType().findFieldByName("value"); sink.add(createDataSetField(name, parentOddr, oddrn, mapType(wrapped.getType()), typeName, true)); return true; } return false; } private static void extractRepeated(Descriptors.FieldDescriptor field, String parentOddr, String oddrn, //null for root String name, boolean nullable, ImmutableSet registeredRecords, List sink) { sink.add(createDataSetField(name, parentOddr, oddrn, TypeEnum.LIST, "repeated", nullable)); String itemName = field.getType() == Descriptors.FieldDescriptor.Type.MESSAGE ? field.getMessageType().getName() : field.getType().name().toLowerCase(); extract( field, oddrn, oddrn + "/items/" + itemName, itemName, nullable, false, registeredRecords, sink ); } private static void extractMessage(Descriptors.FieldDescriptor field, String parentOddr, String oddrn, //null for root String name, boolean nullable, ImmutableSet registeredRecords, List sink) { if (extractProtoWellKnownType(field, parentOddr, oddrn, name, nullable, sink)) { return; } sink.add(createDataSetField(name, parentOddr, oddrn, TypeEnum.STRUCT, getLogicalTypeName(field), nullable)); String msgTypeName = field.getMessageType().getFullName(); if (registeredRecords.contains(msgTypeName)) { // avoiding recursion by checking if record already registered in parsing chain return; } var newRegisteredRecords = ImmutableSet.builder() .addAll(registeredRecords) .add(msgTypeName) .build(); field.getMessageType() .getFields() .forEach(f -> { extract(f, oddrn, oddrn + "/fields/" + f.getName(), f.getName(), !f.isRequired(), f.isRepeated(), newRegisteredRecords, sink ); }); } private static void extractPrimitive(Descriptors.FieldDescriptor field, String parentOddr, String oddrn, String name, boolean nullable, List sink) { sink.add( createDataSetField( name, parentOddr, oddrn, mapType(field.getType()), getLogicalTypeName(field), nullable ) ); } private static String getLogicalTypeName(Descriptors.FieldDescriptor f) { return f.getType() == Descriptors.FieldDescriptor.Type.MESSAGE ? f.getMessageType().getFullName() : f.getType().name().toLowerCase(); } private static DataSetField createDataSetField(String name, String parentOddrn, String oddrn, TypeEnum type, String logicalType, Boolean nullable) { return new DataSetField() .name(name) .parentFieldOddrn(parentOddrn) .oddrn(oddrn) .type( new DataSetFieldType() .isNullable(nullable) .logicalType(logicalType) .type(type) ); } private static TypeEnum mapType(Descriptors.FieldDescriptor.Type type) { return switch (type) { case INT32, INT64, SINT32, SFIXED32, SINT64, UINT32, UINT64, FIXED32, FIXED64, SFIXED64 -> TypeEnum.INTEGER; case FLOAT, DOUBLE -> TypeEnum.NUMBER; case STRING, ENUM -> TypeEnum.STRING; case BOOL -> TypeEnum.BOOLEAN; case BYTES -> TypeEnum.BINARY; case MESSAGE, GROUP -> TypeEnum.STRUCT; }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ksql/KsqlApiClient.java ================================================ package com.provectus.kafka.ui.service.ksql; import static ksql.KsqlGrammarParser.DefineVariableContext; import static ksql.KsqlGrammarParser.PrintTopicContext; import static ksql.KsqlGrammarParser.SingleStatementContext; import static ksql.KsqlGrammarParser.UndefineVariableContext; import static org.springframework.http.MediaType.APPLICATION_JSON; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.TextNode; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.service.ksql.response.ResponseParser; import com.provectus.kafka.ui.util.WebClientConfigurator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; import lombok.Builder; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.springframework.core.codec.DecodingException; import org.springframework.http.MediaType; import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.util.MimeType; import org.springframework.util.MimeTypeUtils; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j public class KsqlApiClient { private static final MimeType KQL_API_MIME_TYPE = MimeTypeUtils.parseMimeType("application/vnd.ksql.v1+json"); private static final Set> UNSUPPORTED_STMT_TYPES = Set.of( PrintTopicContext.class, DefineVariableContext.class, UndefineVariableContext.class ); @Builder(toBuilder = true) @Value public static class KsqlResponseTable { String header; List columnNames; List> values; boolean error; public Optional getColumnValue(List row, String column) { int colIdx = columnNames.indexOf(column); return colIdx >= 0 ? Optional.ofNullable(row.get(colIdx)) : Optional.empty(); } } @Value private static class KsqlRequest { String ksql; Map streamsProperties; } //-------------------------------------------------------------------------------------------- private final String baseUrl; private final WebClient webClient; public KsqlApiClient(String baseUrl, @Nullable ClustersProperties.KsqldbServerAuth ksqldbServerAuth, @Nullable ClustersProperties.TruststoreConfig ksqldbServerSsl, @Nullable ClustersProperties.KeystoreConfig keystoreConfig, @Nullable DataSize maxBuffSize) { this.baseUrl = baseUrl; this.webClient = webClient(ksqldbServerAuth, ksqldbServerSsl, keystoreConfig, maxBuffSize); } private static WebClient webClient(@Nullable ClustersProperties.KsqldbServerAuth ksqldbServerAuth, @Nullable ClustersProperties.TruststoreConfig truststoreConfig, @Nullable ClustersProperties.KeystoreConfig keystoreConfig, @Nullable DataSize maxBuffSize) { ksqldbServerAuth = Optional.ofNullable(ksqldbServerAuth).orElse(new ClustersProperties.KsqldbServerAuth()); maxBuffSize = Optional.ofNullable(maxBuffSize).orElse(DataSize.ofMegabytes(20)); return new WebClientConfigurator() .configureSsl(truststoreConfig, keystoreConfig) .configureBasicAuth( ksqldbServerAuth.getUsername(), ksqldbServerAuth.getPassword() ) .configureBufferSize(maxBuffSize) .configureCodecs(codecs -> { var mapper = new JsonMapper(); codecs.defaultCodecs() .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper, KQL_API_MIME_TYPE, APPLICATION_JSON)); // some ksqldb versions do not set content-type header in response, // but we still need to use JsonDecoder for it codecs.defaultCodecs() .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, MimeTypeUtils.ALL)); }) .build(); } private KsqlRequest ksqlRequest(String ksql, Map streamProperties) { return new KsqlRequest(ksql, streamProperties); } private Flux executeSelect(String ksql, Map streamProperties) { return webClient .post() .uri(baseUrl + "/query") .accept(new MediaType(KQL_API_MIME_TYPE)) .contentType(new MediaType(KQL_API_MIME_TYPE)) .bodyValue(ksqlRequest(ksql, streamProperties)) .retrieve() .bodyToFlux(JsonNode.class) .onErrorResume(this::isUnexpectedJsonArrayEndCharException, th -> Mono.empty()) .map(ResponseParser::parseSelectResponse) .filter(Optional::isPresent) .map(Optional::get) .onErrorResume(WebClientResponseException.class, e -> Flux.just(ResponseParser.parseErrorResponse(e))); } /** * Some version of ksqldb (?..0.24) can cut off json streaming without respect proper array ending like

* [{"header":{"queryId":"...","schema":"..."}}, ] * which will cause json parsing error and will be propagated to UI. * This is a know issue(https://github.com/confluentinc/ksql/issues/8746), but we don't know when it will be fixed. * To workaround this we need to check DecodingException err msg. */ private boolean isUnexpectedJsonArrayEndCharException(Throwable th) { return th instanceof DecodingException && th.getMessage().contains("Unexpected character (']'"); } private Flux executeStatement(String ksql, Map streamProperties) { return webClient .post() .uri(baseUrl + "/ksql") .accept(new MediaType(KQL_API_MIME_TYPE)) .contentType(APPLICATION_JSON) .bodyValue(ksqlRequest(ksql, streamProperties)) .exchangeToFlux( resp -> { if (resp.statusCode().isError()) { return resp.createException().flux().map(ResponseParser::parseErrorResponse); } return resp.bodyToFlux(JsonNode.class) .flatMap(body -> // body can be an array or single object (body.isArray() ? Flux.fromIterable(body) : Flux.just(body)) .flatMapIterable(ResponseParser::parseStatementResponse)) // body can be empty for some statements like INSERT .switchIfEmpty( Flux.just(KsqlResponseTable.builder() .header("Query Result") .columnNames(List.of("Result")) .values(List.of(List.of(new TextNode("Success")))) .build())); } ); } public Flux execute(String ksql, Map streamProperties) { var parsedStatements = KsqlGrammar.parse(ksql); if (parsedStatements.isEmpty()) { return errorTableFlux("Sql statement is invalid or unsupported"); } var statements = parsedStatements.get().getStatements(); if (statements.size() > 1) { return errorTableFlux("Only single statement supported now"); } if (statements.size() == 0) { return errorTableFlux("No valid ksql statement found"); } if (isUnsupportedStatementType(statements.get(0))) { return errorTableFlux("Unsupported statement type"); } Flux outputFlux; if (KsqlGrammar.isSelect(statements.get(0))) { outputFlux = executeSelect(ksql, streamProperties); } else { outputFlux = executeStatement(ksql, streamProperties); } return outputFlux.onErrorResume(Exception.class, e -> { log.error("Unexpected error while execution ksql: {}", ksql, e); return errorTableFlux("Unexpected error: " + e.getMessage()); }); } private Flux errorTableFlux(String errorText) { return Flux.just(ResponseParser.errorTableWithTextMsg(errorText)); } private boolean isUnsupportedStatementType(SingleStatementContext context) { var ctxClass = context.statement().getClass(); return UNSUPPORTED_STMT_TYPES.contains(ctxClass); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ksql/KsqlGrammar.java ================================================ package com.provectus.kafka.ui.service.ksql; import com.provectus.kafka.ui.exception.ValidationException; import java.util.List; import java.util.Optional; import ksql.KsqlGrammarLexer; import ksql.KsqlGrammarParser; import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.experimental.Delegate; import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.atn.PredictionMode; class KsqlGrammar { private KsqlGrammar() { } @Value static class KsqlStatements { List statements; } // returns Empty if no valid statements found static Optional parse(String ksql) { var parsed = parseStatements(ksql); if (parsed.singleStatement().stream() .anyMatch(s -> s.statement().exception != null)) { return Optional.empty(); } return Optional.of(new KsqlStatements(parsed.singleStatement())); } static boolean isSelect(KsqlGrammarParser.SingleStatementContext statement) { return statement.statement() instanceof ksql.KsqlGrammarParser.QueryStatementContext; } private static ksql.KsqlGrammarParser.StatementsContext parseStatements(final String sql) { var lexer = new KsqlGrammarLexer(CaseInsensitiveStream.from(CharStreams.fromString(sql))); var tokenStream = new CommonTokenStream(lexer); var grammarParser = new ksql.KsqlGrammarParser(tokenStream); lexer.addErrorListener(new BaseErrorListener() { @Override public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { throw new ValidationException("Invalid syntax: " + msg); } }); grammarParser.getInterpreter().setPredictionMode(PredictionMode.LL); try { return grammarParser.statements(); } catch (Exception e) { throw new ValidationException("Error parsing ksql query: " + e.getMessage()); } } // impl copied from https://github.com/confluentinc/ksql/blob/master/ksqldb-parser/src/main/java/io/confluent/ksql/parser/CaseInsensitiveStream.java @RequiredArgsConstructor private static class CaseInsensitiveStream implements CharStream { @Delegate final CharStream stream; public static CaseInsensitiveStream from(CharStream stream) { // we only need to override LA method return new CaseInsensitiveStream(stream) { @Override public int LA(final int i) { final int result = stream.LA(i); switch (result) { case 0: case IntStream.EOF: return result; default: return Character.toUpperCase(result); } } }; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2.java ================================================ package com.provectus.kafka.ui.service.ksql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.provectus.kafka.ui.exception.KsqlApiException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.KsqlStreamDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableDescriptionDTO; import com.provectus.kafka.ui.service.ksql.KsqlApiClient.KsqlResponseTable; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; @Slf4j @Service public class KsqlServiceV2 { @lombok.Value private static class KsqlExecuteCommand { KafkaCluster cluster; String ksql; Map streamProperties; } private final Cache registeredCommands = CacheBuilder.newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) .build(); public String registerCommand(KafkaCluster cluster, String ksql, Map streamProperties) { String uuid = UUID.randomUUID().toString(); registeredCommands.put(uuid, new KsqlExecuteCommand(cluster, ksql, streamProperties)); return uuid; } public Flux execute(String commandId) { var cmd = registeredCommands.getIfPresent(commandId); if (cmd == null) { throw new ValidationException("No command registered with id " + commandId); } registeredCommands.invalidate(commandId); return cmd.cluster.getKsqlClient() .flux(client -> client.execute(cmd.ksql, cmd.streamProperties)); } public Flux listTables(KafkaCluster cluster) { return cluster.getKsqlClient() .flux(client -> client.execute("LIST TABLES;", Map.of())) .flatMap(resp -> { if (!resp.getHeader().equals("Tables")) { log.error("Unexpected result header: {}", resp.getHeader()); log.debug("Unexpected result {}", resp); return Flux.error(new KsqlApiException("Error retrieving tables list")); } return Flux.fromIterable(resp.getValues() .stream() .map(row -> new KsqlTableDescriptionDTO() .name(resp.getColumnValue(row, "name").map(JsonNode::asText).orElse(null)) .topic(resp.getColumnValue(row, "topic").map(JsonNode::asText).orElse(null)) .keyFormat(resp.getColumnValue(row, "keyFormat").map(JsonNode::asText).orElse(null)) .valueFormat(resp.getColumnValue(row, "valueFormat").map(JsonNode::asText).orElse(null)) .isWindowed(resp.getColumnValue(row, "isWindowed").map(JsonNode::asBoolean).orElse(null))) .collect(Collectors.toList())); }); } public Flux listStreams(KafkaCluster cluster) { return cluster.getKsqlClient() .flux(client -> client.execute("LIST STREAMS;", Map.of())) .flatMap(resp -> { if (!resp.getHeader().equals("Streams")) { log.error("Unexpected result header: {}", resp.getHeader()); log.debug("Unexpected result {}", resp); return Flux.error(new KsqlApiException("Error retrieving streams list")); } return Flux.fromIterable(resp.getValues() .stream() .map(row -> new KsqlStreamDescriptionDTO() .name(resp.getColumnValue(row, "name").map(JsonNode::asText).orElse(null)) .topic(resp.getColumnValue(row, "topic").map(JsonNode::asText).orElse(null)) .keyFormat(resp.getColumnValue(row, "keyFormat").map(JsonNode::asText).orElse(null)) .valueFormat( // for old versions (<0.13) "format" column is filled, // for new version "keyFormat" & "valueFormat" columns should be filled resp.getColumnValue(row, "valueFormat") .or(() -> resp.getColumnValue(row, "format")) .map(JsonNode::asText) .orElse(null)) ) .collect(Collectors.toList())); }); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ksql/response/DynamicParser.java ================================================ package com.provectus.kafka.ui.service.ksql.response; import static com.provectus.kafka.ui.service.ksql.KsqlApiClient.KsqlResponseTable; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; class DynamicParser { private DynamicParser() { } static KsqlResponseTable parseArray(String tableName, JsonNode array) { return parseArray(tableName, getFieldNamesFromArray(array), array); } static KsqlResponseTable parseArray(String tableName, List columnNames, JsonNode array) { return KsqlResponseTable.builder() .header(tableName) .columnNames(columnNames) .values( StreamSupport.stream(array.spliterator(), false) .map(node -> columnNames.stream() .map(node::get) .collect(Collectors.toList())) .collect(Collectors.toList()) ).build(); } private static List getFieldNamesFromArray(JsonNode array) { List fields = new ArrayList<>(); array.forEach(node -> node.fieldNames().forEachRemaining(f -> { if (!fields.contains(f)) { fields.add(f); } })); return fields; } static KsqlResponseTable parseObject(String tableName, JsonNode node) { if (!node.isObject()) { return KsqlResponseTable.builder() .header(tableName) .columnNames(List.of("value")) .values(List.of(List.of(node))) .build(); } return parseObject(tableName, Lists.newArrayList(node.fieldNames()), node); } static KsqlResponseTable parseObject(String tableName, List columnNames, JsonNode node) { return KsqlResponseTable.builder() .header(tableName) .columnNames(columnNames) .values( List.of( columnNames.stream() .map(node::get) .collect(Collectors.toList())) ) .build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ksql/response/ResponseParser.java ================================================ package com.provectus.kafka.ui.service.ksql.response; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.provectus.kafka.ui.exception.KsqlApiException; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.springframework.web.reactive.function.client.WebClientResponseException; public class ResponseParser { private ResponseParser() { } public static Optional parseSelectResponse(JsonNode jsonNode) { // in response, we're getting either header record or row data if (arrayFieldNonEmpty(jsonNode, "header")) { return Optional.of( KsqlApiClient.KsqlResponseTable.builder() .header("Schema") .columnNames(parseSelectHeadersString(jsonNode.get("header").get("schema").asText())) .build()); } if (arrayFieldNonEmpty(jsonNode, "row")) { return Optional.of( KsqlApiClient.KsqlResponseTable.builder() .header("Row") .values( List.of(Lists.newArrayList(jsonNode.get("row").get("columns")))) .build()); } if (jsonNode.hasNonNull("errorMessage")) { throw new KsqlApiException("Error: " + jsonNode.get("errorMessage")); } // remaining events can be skipped return Optional.empty(); } @VisibleForTesting static List parseSelectHeadersString(String str) { List headers = new ArrayList<>(); int structNesting = 0; boolean quotes = false; var headerBuilder = new StringBuilder(); for (char ch : str.toCharArray()) { if (ch == '<') { structNesting++; } else if (ch == '>') { structNesting--; } else if (ch == '`') { quotes = !quotes; } else if (ch == ' ' && headerBuilder.isEmpty()) { continue; //skipping leading & training whitespaces } else if (ch == ',' && structNesting == 0 && !quotes) { headers.add(headerBuilder.toString()); headerBuilder = new StringBuilder(); continue; } headerBuilder.append(ch); } if (!headerBuilder.isEmpty()) { headers.add(headerBuilder.toString()); } return headers; } public static KsqlApiClient.KsqlResponseTable errorTableWithTextMsg(String errorText) { return KsqlApiClient.KsqlResponseTable.builder() .header("Execution error") .columnNames(List.of("message")) .values(List.of(List.of(new TextNode(errorText)))) .error(true) .build(); } public static KsqlApiClient.KsqlResponseTable parseErrorResponse(WebClientResponseException e) { try { var errBody = new JsonMapper().readTree(e.getResponseBodyAsString()); return DynamicParser.parseObject("Execution error", errBody) .toBuilder() .error(true) .build(); } catch (Exception ex) { return errorTableWithTextMsg( String.format( "Unparsable error response from ksqdb, status:'%s', body: '%s'", e.getStatusCode(), e.getResponseBodyAsString())); } } public static List parseStatementResponse(JsonNode jsonNode) { var type = Optional.ofNullable(jsonNode.get("@type")) .map(JsonNode::asText) .orElse("unknown"); // messages structure can be inferred from https://github.com/confluentinc/ksql/blob/master/ksqldb-rest-model/src/main/java/io/confluent/ksql/rest/entity/KsqlEntity.java switch (type) { case "currentStatus": return parseObject( "Status", List.of("status", "message"), jsonNode.get("commandStatus") ); case "properties": return parseProperties(jsonNode); case "queries": return parseArray("Queries", "queries", jsonNode); case "sourceDescription": return parseObjectDynamically("Source Description", jsonNode.get("sourceDescription")); case "queryDescription": return parseObjectDynamically("Queries Description", jsonNode.get("queryDescription")); case "topicDescription": return parseObject( "Topic Description", List.of("name", "kafkaTopic", "format", "schemaString"), jsonNode ); case "streams": return parseArray("Streams", "streams", jsonNode); case "tables": return parseArray("Tables", "tables", jsonNode); case "kafka_topics": return parseArray("Topics", "topics", jsonNode); case "kafka_topics_extended": return parseArray("Topics extended", "topics", jsonNode); case "executionPlan": return parseObject("Execution plan", List.of("executionPlanText"), jsonNode); case "source_descriptions": return parseArray("Source descriptions", "sourceDescriptions", jsonNode); case "query_descriptions": return parseArray("Queries", "queryDescriptions", jsonNode); case "describe_function": return parseObject("Function description", List.of("name", "author", "version", "description", "functions", "path", "type"), jsonNode ); case "function_names": return parseArray("Function Names", "functions", jsonNode); case "connector_info": return parseObjectDynamically("Connector Info", jsonNode.get("info")); case "drop_connector": return parseObject("Dropped connector", List.of("connectorName"), jsonNode); case "connector_list": return parseArray("Connectors", "connectors", jsonNode); case "connector_plugins_list": return parseArray("Connector Plugins", "connectorPlugins", jsonNode); case "connector_description": return parseObject("Connector Description", List.of("connectorClass", "status", "sources", "topics"), jsonNode ); default: return parseUnknownResponse(jsonNode); } } private static List parseObjectDynamically( String tableName, JsonNode jsonNode) { return List.of(DynamicParser.parseObject(tableName, jsonNode)); } private static List parseObject( String tableName, List fields, JsonNode jsonNode) { return List.of(DynamicParser.parseObject(tableName, fields, jsonNode)); } private static List parseArray( String tableName, String arrayField, JsonNode jsonNode) { return List.of(DynamicParser.parseArray(tableName, jsonNode.get(arrayField))); } private static List parseProperties(JsonNode jsonNode) { var tables = new ArrayList(); if (arrayFieldNonEmpty(jsonNode, "properties")) { tables.add(DynamicParser.parseArray("properties", jsonNode.get("properties"))); } if (arrayFieldNonEmpty(jsonNode, "overwrittenProperties")) { tables.add(DynamicParser.parseArray("overwrittenProperties", jsonNode.get("overwrittenProperties"))); } return tables; } private static List parseUnknownResponse(JsonNode jsonNode) { return List.of(DynamicParser.parseObject("Ksql Response", jsonNode)); } private static boolean arrayFieldNonEmpty(JsonNode json, String field) { return json.hasNonNull(field) && !json.get(field).isEmpty(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/DataMasking.java ================================================ package com.provectus.kafka.ui.service.masking; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.service.masking.policies.MaskingPolicy; import java.util.List; import java.util.Optional; import java.util.function.UnaryOperator; import java.util.regex.Pattern; import javax.annotation.Nullable; import lombok.Value; import org.apache.commons.lang3.StringUtils; public class DataMasking { private static final JsonMapper JSON_MAPPER = new JsonMapper(); @Value static class Mask { @Nullable Pattern topicKeysPattern; @Nullable Pattern topicValuesPattern; MaskingPolicy policy; boolean shouldBeApplied(String topic, Serde.Target target) { return target == Serde.Target.KEY ? topicKeysPattern != null && topicKeysPattern.matcher(topic).matches() : topicValuesPattern != null && topicValuesPattern.matcher(topic).matches(); } } private final List masks; public static DataMasking create(@Nullable List config) { return new DataMasking( Optional.ofNullable(config).orElse(List.of()).stream().map(property -> { Preconditions.checkNotNull(property.getType(), "masking type not specified"); Preconditions.checkArgument( StringUtils.isNotEmpty(property.getTopicKeysPattern()) || StringUtils.isNotEmpty(property.getTopicValuesPattern()), "topicKeysPattern or topicValuesPattern (or both) should be set for masking policy"); return new Mask( Optional.ofNullable(property.getTopicKeysPattern()).map(Pattern::compile).orElse(null), Optional.ofNullable(property.getTopicValuesPattern()).map(Pattern::compile).orElse(null), MaskingPolicy.create(property) ); }).toList() ); } @VisibleForTesting DataMasking(List masks) { this.masks = masks; } public UnaryOperator getMaskerForTopic(String topic) { var keyMasker = getMaskingFunction(topic, Serde.Target.KEY); var valMasker = getMaskingFunction(topic, Serde.Target.VALUE); return msg -> msg .key(keyMasker.apply(msg.getKey())) .content(valMasker.apply(msg.getContent())); } @VisibleForTesting UnaryOperator getMaskingFunction(String topic, Serde.Target target) { var targetMasks = masks.stream().filter(m -> m.shouldBeApplied(topic, target)).toList(); if (targetMasks.isEmpty()) { return UnaryOperator.identity(); } return inputStr -> { if (inputStr == null) { return null; } try { JsonNode json = JSON_MAPPER.readTree(inputStr); if (json.isContainerNode()) { for (Mask targetMask : targetMasks) { json = targetMask.policy.applyToJsonContainer((ContainerNode) json); } return json.toString(); } } catch (JsonProcessingException jsonException) { //just ignore } // if we can't parse input as json or parsed json is not object/array // we just apply first found policy // (there is no need to apply all of them, because they will just override each other) return targetMasks.get(0).policy.applyToString(inputStr); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelector.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.exception.ValidationException; import java.util.regex.Pattern; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; interface FieldsSelector { static FieldsSelector create(ClustersProperties.Masking property) { if (StringUtils.hasText(property.getFieldsNamePattern()) && !CollectionUtils.isEmpty(property.getFields())) { throw new ValidationException("You can't provide both fieldNames & fieldsNamePattern for masking"); } if (StringUtils.hasText(property.getFieldsNamePattern())) { Pattern pattern = Pattern.compile(property.getFieldsNamePattern()); return f -> pattern.matcher(f).matches(); } if (!CollectionUtils.isEmpty(property.getFields())) { return f -> property.getFields().contains(f); } //no pattern, no field names - mean all fields should be masked return fieldName -> true; } boolean shouldBeMasked(String fieldName); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Mask.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Preconditions; import java.util.List; import java.util.function.UnaryOperator; class Mask extends MaskingPolicy { static final List DEFAULT_PATTERN = List.of("X", "x", "n", "-"); private final UnaryOperator masker; Mask(FieldsSelector fieldsSelector, List maskingChars) { super(fieldsSelector); this.masker = createMasker(maskingChars); } @Override public ContainerNode applyToJsonContainer(ContainerNode node) { return (ContainerNode) maskWithFieldsCheck(node); } @Override public String applyToString(String str) { return masker.apply(str); } private static UnaryOperator createMasker(List maskingChars) { Preconditions.checkNotNull(maskingChars); Preconditions.checkArgument(maskingChars.size() == 4, "mask pattern should contain 4 elements"); return input -> { StringBuilder sb = new StringBuilder(input.length()); for (int i = 0; i < input.length(); i++) { int cp = input.codePointAt(i); switch (Character.getType(cp)) { case Character.SPACE_SEPARATOR, Character.LINE_SEPARATOR, Character.PARAGRAPH_SEPARATOR -> sb.appendCodePoint(cp); // keeping separators as-is case Character.UPPERCASE_LETTER -> sb.append(maskingChars.get(0)); case Character.LOWERCASE_LETTER -> sb.append(maskingChars.get(1)); case Character.DECIMAL_DIGIT_NUMBER -> sb.append(maskingChars.get(2)); default -> sb.append(maskingChars.get(3)); } } return sb.toString(); }; } private JsonNode maskWithFieldsCheck(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); node.fields().forEachRemaining(f -> { String fieldName = f.getKey(); JsonNode fieldVal = f.getValue(); if (fieldShouldBeMasked(fieldName)) { obj.set(fieldName, maskNodeRecursively(fieldVal)); } else { obj.set(fieldName, maskWithFieldsCheck(fieldVal)); } }); return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); node.elements().forEachRemaining(e -> arr.add(maskWithFieldsCheck(e))); return arr; } return node; } private JsonNode maskNodeRecursively(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); node.fields().forEachRemaining(f -> obj.set(f.getKey(), maskNodeRecursively(f.getValue()))); return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); node.elements().forEachRemaining(e -> arr.add(maskNodeRecursively(e))); return arr; } return new TextNode(masker.apply(node.asText())); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/MaskingPolicy.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import com.fasterxml.jackson.databind.node.ContainerNode; import com.provectus.kafka.ui.config.ClustersProperties; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public abstract class MaskingPolicy { public static MaskingPolicy create(ClustersProperties.Masking property) { FieldsSelector fieldsSelector = FieldsSelector.create(property); return switch (property.getType()) { case REMOVE -> new Remove(fieldsSelector); case REPLACE -> new Replace( fieldsSelector, property.getReplacement() == null ? Replace.DEFAULT_REPLACEMENT : property.getReplacement() ); case MASK -> new Mask( fieldsSelector, property.getMaskingCharsReplacement() == null ? Mask.DEFAULT_PATTERN : property.getMaskingCharsReplacement() ); }; } //---------------------------------------------------------------- private final FieldsSelector fieldsSelector; protected boolean fieldShouldBeMasked(String fieldName) { return fieldsSelector.shouldBeMasked(fieldName); } public abstract ContainerNode applyToJsonContainer(ContainerNode node); public abstract String applyToString(String str); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Remove.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; class Remove extends MaskingPolicy { Remove(FieldsSelector fieldsSelector) { super(fieldsSelector); } @Override public String applyToString(String str) { return "null"; } @Override public ContainerNode applyToJsonContainer(ContainerNode node) { return (ContainerNode) removeFields(node); } private JsonNode removeFields(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); node.fields().forEachRemaining(f -> { String fieldName = f.getKey(); JsonNode fieldVal = f.getValue(); if (!fieldShouldBeMasked(fieldName)) { obj.set(fieldName, removeFields(fieldVal)); } }); return obj; } else if (node.isArray()) { var arr = ((ArrayNode) node).arrayNode(node.size()); node.elements().forEachRemaining(e -> arr.add(removeFields(e))); return arr; } return node; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/masking/policies/Replace.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Preconditions; class Replace extends MaskingPolicy { static final String DEFAULT_REPLACEMENT = "***DATA_MASKED***"; private final String replacement; Replace(FieldsSelector fieldsSelector, String replacementString) { super(fieldsSelector); this.replacement = Preconditions.checkNotNull(replacementString); } @Override public String applyToString(String str) { return replacement; } @Override public ContainerNode applyToJsonContainer(ContainerNode node) { return (ContainerNode) replaceWithFieldsCheck(node); } private JsonNode replaceWithFieldsCheck(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); node.fields().forEachRemaining(f -> { String fieldName = f.getKey(); JsonNode fieldVal = f.getValue(); if (fieldShouldBeMasked(fieldName)) { obj.set(fieldName, replaceRecursive(fieldVal)); } else { obj.set(fieldName, replaceWithFieldsCheck(fieldVal)); } }); return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); node.elements().forEachRemaining(e -> arr.add(replaceWithFieldsCheck(e))); return arr; } // if it is not an object or array - we have nothing to replace here return node; } private JsonNode replaceRecursive(JsonNode node) { if (node.isObject()) { ObjectNode obj = ((ObjectNode) node).objectNode(); node.fields().forEachRemaining(f -> obj.set(f.getKey(), replaceRecursive(f.getValue()))); return obj; } else if (node.isArray()) { ArrayNode arr = ((ArrayNode) node).arrayNode(node.size()); node.elements().forEachRemaining(e -> arr.add(replaceRecursive(e))); return arr; } return new TextNode(replacement); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatter.java ================================================ package com.provectus.kafka.ui.service.metrics; import java.math.BigDecimal; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.management.MBeanAttributeInfo; import javax.management.ObjectName; /** * Converts JMX metrics into JmxExporter prometheus format: format. */ class JmxMetricsFormatter { // copied from https://github.com/prometheus/jmx_exporter/blob/b6b811b4aae994e812e902b26dd41f29364c0e2b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java#L15 private static final Pattern PROPERTY_PATTERN = Pattern.compile( "([^,=:\\*\\?]+)=(\"(?:[^\\\\\"]*(?:\\\\.)?)*\"|[^,=:\"]*)"); static List constructMetricsList(ObjectName jmxMetric, MBeanAttributeInfo[] attributes, Object[] attrValues) { String domain = fixIllegalChars(jmxMetric.getDomain()); LinkedHashMap labels = getLabelsMap(jmxMetric); String firstLabel = labels.keySet().iterator().next(); String firstLabelValue = fixIllegalChars(labels.get(firstLabel)); labels.remove(firstLabel); //removing first label since it's value will be in name List result = new ArrayList<>(attributes.length); for (int i = 0; i < attributes.length; i++) { String attrName = fixIllegalChars(attributes[i].getName()); convertNumericValue(attrValues[i]).ifPresent(convertedValue -> { String name = String.format("%s_%s_%s", domain, firstLabelValue, attrName); var metric = RawMetric.create(name, labels, convertedValue); result.add(metric); }); } return result; } private static String fixIllegalChars(String str) { return str .replace('.', '_') .replace('-', '_'); } private static Optional convertNumericValue(Object value) { if (!(value instanceof Number)) { return Optional.empty(); } try { if (value instanceof Long) { return Optional.of(new BigDecimal((Long) value)); } else if (value instanceof Integer) { return Optional.of(new BigDecimal((Integer) value)); } return Optional.of(new BigDecimal(value.toString())); } catch (NumberFormatException nfe) { return Optional.empty(); } } /** * Converts Mbean properties to map keeping order (copied from jmx_exporter repo). */ private static LinkedHashMap getLabelsMap(ObjectName mbeanName) { LinkedHashMap keyProperties = new LinkedHashMap<>(); String properties = mbeanName.getKeyPropertyListString(); Matcher match = PROPERTY_PATTERN.matcher(properties); while (match.lookingAt()) { String labelName = fixIllegalChars(match.group(1)); // label names should be fixed String labelValue = match.group(2); keyProperties.put(labelName, labelValue); properties = properties.substring(match.end()); if (properties.startsWith(",")) { properties = properties.substring(1); } match.reset(properties); } return keyProperties; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxMetricsRetriever.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.provectus.kafka.ui.model.KafkaCluster; import java.io.Closeable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; import javax.management.MBeanAttributeInfo; import javax.management.MBeanServerConnection; import javax.management.ObjectName; import javax.management.remote.JMXConnector; import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.common.Node; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @Service @Slf4j class JmxMetricsRetriever implements MetricsRetriever, Closeable { private static final boolean SSL_JMX_SUPPORTED; static { // see JmxSslSocketFactory doc for details SSL_JMX_SUPPORTED = JmxSslSocketFactory.initialized(); } private static final String JMX_URL = "service:jmx:rmi:///jndi/rmi://"; private static final String JMX_SERVICE_TYPE = "jmxrmi"; private static final String CANONICAL_NAME_PATTERN = "kafka.server*:*"; @Override public void close() { JmxSslSocketFactory.clearFactoriesCache(); } @Override public Flux retrieve(KafkaCluster c, Node node) { if (isSslJmxEndpoint(c) && !SSL_JMX_SUPPORTED) { log.warn("Cluster {} has jmx ssl configured, but it is not supported", c.getName()); return Flux.empty(); } return Mono.fromSupplier(() -> retrieveSync(c, node)) .subscribeOn(Schedulers.boundedElastic()) .flatMapMany(Flux::fromIterable); } private boolean isSslJmxEndpoint(KafkaCluster cluster) { return cluster.getMetricsConfig().getKeystoreLocation() != null; } @SneakyThrows private List retrieveSync(KafkaCluster c, Node node) { String jmxUrl = JMX_URL + node.host() + ":" + c.getMetricsConfig().getPort() + "/" + JMX_SERVICE_TYPE; log.debug("Collection JMX metrics for {}", jmxUrl); List result = new ArrayList<>(); withJmxConnector(jmxUrl, c, jmxConnector -> getMetricsFromJmx(jmxConnector, result)); log.debug("{} metrics collected for {}", result.size(), jmxUrl); return result; } private void withJmxConnector(String jmxUrl, KafkaCluster c, Consumer consumer) { var env = prepareJmxEnvAndSetThreadLocal(c); try (JMXConnector connector = JMXConnectorFactory.newJMXConnector(new JMXServiceURL(jmxUrl), env)) { try { connector.connect(env); } catch (Exception exception) { log.error("Error connecting to {}", jmxUrl, exception); return; } consumer.accept(connector); } catch (Exception e) { log.error("Error getting jmx metrics from {}", jmxUrl, e); } finally { JmxSslSocketFactory.clearThreadLocalContext(); } } private Map prepareJmxEnvAndSetThreadLocal(KafkaCluster cluster) { var metricsConfig = cluster.getMetricsConfig(); Map env = new HashMap<>(); if (isSslJmxEndpoint(cluster)) { var clusterSsl = cluster.getOriginalProperties().getSsl(); JmxSslSocketFactory.setSslContextThreadLocal( clusterSsl != null ? clusterSsl.getTruststoreLocation() : null, clusterSsl != null ? clusterSsl.getTruststorePassword() : null, metricsConfig.getKeystoreLocation(), metricsConfig.getKeystorePassword() ); JmxSslSocketFactory.editJmxConnectorEnv(env); } if (StringUtils.isNotEmpty(metricsConfig.getUsername()) && StringUtils.isNotEmpty(metricsConfig.getPassword())) { env.put( JMXConnector.CREDENTIALS, new String[] {metricsConfig.getUsername(), metricsConfig.getPassword()} ); } return env; } @SneakyThrows private void getMetricsFromJmx(JMXConnector jmxConnector, List sink) { MBeanServerConnection msc = jmxConnector.getMBeanServerConnection(); var jmxMetrics = msc.queryNames(new ObjectName(CANONICAL_NAME_PATTERN), null); for (ObjectName jmxMetric : jmxMetrics) { sink.addAll(extractObjectMetrics(jmxMetric, msc)); } } @SneakyThrows private List extractObjectMetrics(ObjectName objectName, MBeanServerConnection msc) { MBeanAttributeInfo[] attrNames = msc.getMBeanInfo(objectName).getAttributes(); Object[] attrValues = new Object[attrNames.length]; for (int i = 0; i < attrNames.length; i++) { attrValues[i] = msc.getAttribute(objectName, attrNames[i].getName()); } return JmxMetricsFormatter.constructMetricsList(objectName, attrNames, attrValues); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/JmxSslSocketFactory.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.google.common.base.Preconditions; import java.io.FileInputStream; import java.io.IOException; import java.lang.reflect.Field; import java.net.InetAddress; import java.net.Socket; import java.net.UnknownHostException; import java.security.KeyStore; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nullable; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; import javax.rmi.ssl.SslRMIClientSocketFactory; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.util.ResourceUtils; /* * Purpose of this class to provide an ability to connect to different JMX endpoints using different keystores. * * Usually, when you want to establish SSL JMX connection you set "com.sun.jndi.rmi.factory.socket" env * property to SslRMIClientSocketFactory instance. SslRMIClientSocketFactory itself uses SSLSocketFactory.getDefault() * as a socket factory implementation. Problem here is that when ones SslRMIClientSocketFactory instance is created, * the same cached SSLSocketFactory instance will be used to establish connection with *all* JMX endpoints. * Moreover, even if we submit custom SslRMIClientSocketFactory implementation which takes specific ssl context * into account, SslRMIClientSocketFactory is * internally created during RMI calls. * * So, the only way we found to deal with it is to change internal field ('defaultSocketFactory') of * SslRMIClientSocketFactory to our custom impl, and left all internal RMI code work as is. * Since RMI code is synchronous, we can pass parameters (which are truststore/keystore) to our custom factory * that we want to use when creating ssl socket via ThreadLocal variables. * * NOTE 1: Theoretically we could avoid using reflection to set internal field set by * setting "ssl.SocketFactory.provider" security property (see code in SSLSocketFactory.getDefault()), * but that code uses systemClassloader which is not working right when we're creating executable spring boot jar * (https://docs.spring.io/spring-boot/docs/current/reference/html/executable-jar.html#appendix.executable-jar.restrictions). * We can use this if we swith to other jar-packing solutions in the future. * * NOTE 2: There are two paths from which socket factory is called - when jmx connection if established (we manage this * by passing ThreadLocal vars) and from DGCClient in background thread - we deal with that we cache created factories * for specific host+port. * */ @Slf4j class JmxSslSocketFactory extends javax.net.ssl.SSLSocketFactory { private static final boolean SSL_JMX_SUPPORTED; static { boolean sslJmxSupported = false; try { Field defaultSocketFactoryField = SslRMIClientSocketFactory.class.getDeclaredField("defaultSocketFactory"); defaultSocketFactoryField.setAccessible(true); defaultSocketFactoryField.set(null, new JmxSslSocketFactory()); sslJmxSupported = true; } catch (Exception e) { log.error("----------------------------------"); log.error("SSL can't be enabled for JMX retrieval. " + "Make sure your java app run with '--add-opens java.rmi/javax.rmi.ssl=ALL-UNNAMED' arg. Err: {}", e.getMessage()); log.trace("SSL can't be enabled for JMX retrieval", e); log.error("----------------------------------"); } SSL_JMX_SUPPORTED = sslJmxSupported; } public static boolean initialized() { return SSL_JMX_SUPPORTED; } private static final ThreadLocal SSL_CONTEXT_THREAD_LOCAL = new ThreadLocal<>(); private static final Map CACHED_FACTORIES = new ConcurrentHashMap<>(); private record HostAndPort(String host, int port) { } private record Ssl(@Nullable String truststoreLocation, @Nullable String truststorePassword, @Nullable String keystoreLocation, @Nullable String keystorePassword) { } public static void setSslContextThreadLocal(@Nullable String truststoreLocation, @Nullable String truststorePassword, @Nullable String keystoreLocation, @Nullable String keystorePassword) { SSL_CONTEXT_THREAD_LOCAL.set( new Ssl(truststoreLocation, truststorePassword, keystoreLocation, keystorePassword)); } // should be called when (host:port) -> factory cache should be invalidated (ex. on app config reload) public static void clearFactoriesCache() { CACHED_FACTORIES.clear(); } public static void clearThreadLocalContext() { SSL_CONTEXT_THREAD_LOCAL.set(null); } public static void editJmxConnectorEnv(Map env) { env.put("com.sun.jndi.rmi.factory.socket", new SslRMIClientSocketFactory()); } //----------------------------------------------------------------------------------------------- private final javax.net.ssl.SSLSocketFactory defaultSocketFactory; @SneakyThrows public JmxSslSocketFactory() { this.defaultSocketFactory = SSLContext.getDefault().getSocketFactory(); } @SneakyThrows private javax.net.ssl.SSLSocketFactory createFactoryFromThreadLocalCtx() { Ssl ssl = Preconditions.checkNotNull(SSL_CONTEXT_THREAD_LOCAL.get()); var trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); if (ssl.truststoreLocation() != null && ssl.truststorePassword() != null) { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load( new FileInputStream((ResourceUtils.getFile(ssl.truststoreLocation()))), ssl.truststorePassword().toCharArray() ); trustManagerFactory.init(trustStore); } var keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); if (ssl.keystoreLocation() != null && ssl.keystorePassword() != null) { KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); keyStore.load( new FileInputStream(ResourceUtils.getFile(ssl.keystoreLocation())), ssl.keystorePassword().toCharArray() ); keyManagerFactory.init(keyStore, ssl.keystorePassword().toCharArray()); } SSLContext ctx = SSLContext.getInstance("TLS"); ctx.init( keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null ); return ctx.getSocketFactory(); } private boolean threadLocalContextSet() { return SSL_CONTEXT_THREAD_LOCAL.get() != null; } @Override public Socket createSocket(String host, int port) throws IOException { var hostAndPort = new HostAndPort(host, port); if (CACHED_FACTORIES.containsKey(hostAndPort)) { return CACHED_FACTORIES.get(hostAndPort).createSocket(host, port); } else if (threadLocalContextSet()) { var factory = createFactoryFromThreadLocalCtx(); CACHED_FACTORIES.put(hostAndPort, factory); return factory.createSocket(host, port); } return defaultSocketFactory.createSocket(host, port); } /// FOLLOWING METHODS WON'T BE USED DURING JMX INTERACTION, IMPLEMENTING THEM JUST FOR CONSISTENCY ->>>>> @Override public Socket createSocket(Socket s, String host, int port, boolean autoClose) throws IOException { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().createSocket(s, host, port, autoClose); } return defaultSocketFactory.createSocket(s, host, port, autoClose); } @Override public Socket createSocket(String host, int port, InetAddress localHost, int localPort) throws IOException, UnknownHostException { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().createSocket(host, port, localHost, localPort); } return defaultSocketFactory.createSocket(host, port, localHost, localPort); } @Override public Socket createSocket(InetAddress host, int port) throws IOException { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().createSocket(host, port); } return defaultSocketFactory.createSocket(host, port); } @Override public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort) throws IOException { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().createSocket(address, port, localAddress, localPort); } return defaultSocketFactory.createSocket(address, port, localAddress, localPort); } @Override public String[] getDefaultCipherSuites() { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().getDefaultCipherSuites(); } return defaultSocketFactory.getDefaultCipherSuites(); } @Override public String[] getSupportedCipherSuites() { if (threadLocalContextSet()) { return createFactoryFromThreadLocalCtx().getSupportedCipherSuites(); } return defaultSocketFactory.getSupportedCipherSuites(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsCollector.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.MetricsConfig; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.Node; import org.springframework.stereotype.Component; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Component @Slf4j @RequiredArgsConstructor public class MetricsCollector { private final JmxMetricsRetriever jmxMetricsRetriever; private final PrometheusMetricsRetriever prometheusMetricsRetriever; public Mono getBrokerMetrics(KafkaCluster cluster, Collection nodes) { return Flux.fromIterable(nodes) .flatMap(n -> getMetrics(cluster, n).map(lst -> Tuples.of(n, lst))) .collectMap(Tuple2::getT1, Tuple2::getT2) .map(nodeMetrics -> collectMetrics(cluster, nodeMetrics)) .defaultIfEmpty(Metrics.empty()); } private Mono> getMetrics(KafkaCluster kafkaCluster, Node node) { Flux metricFlux = Flux.empty(); if (kafkaCluster.getMetricsConfig() != null) { String type = kafkaCluster.getMetricsConfig().getType(); if (type == null || type.equalsIgnoreCase(MetricsConfig.JMX_METRICS_TYPE)) { metricFlux = jmxMetricsRetriever.retrieve(kafkaCluster, node); } else if (type.equalsIgnoreCase(MetricsConfig.PROMETHEUS_METRICS_TYPE)) { metricFlux = prometheusMetricsRetriever.retrieve(kafkaCluster, node); } } return metricFlux.collectList(); } public Metrics collectMetrics(KafkaCluster cluster, Map> perBrokerMetrics) { Metrics.MetricsBuilder builder = Metrics.builder() .perBrokerMetrics( perBrokerMetrics.entrySet() .stream() .collect(Collectors.toMap(e -> e.getKey().id(), Map.Entry::getValue))); populateWellknowMetrics(cluster, perBrokerMetrics) .apply(builder); return builder.build(); } private WellKnownMetrics populateWellknowMetrics(KafkaCluster cluster, Map> perBrokerMetrics) { WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); perBrokerMetrics.forEach((node, metrics) -> metrics.forEach(metric -> wellKnownMetrics.populate(node, metric))); return wellKnownMetrics; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/MetricsRetriever.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.provectus.kafka.ui.model.KafkaCluster; import org.apache.kafka.common.Node; import reactor.core.publisher.Flux; interface MetricsRetriever { Flux retrieve(KafkaCluster c, Node node); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParser.java ================================================ package com.provectus.kafka.ui.service.metrics; import java.math.BigDecimal; import java.util.Arrays; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; @Slf4j class PrometheusEndpointMetricsParser { /** * Matches openmetrics format. For example, string: * kafka_server_BrokerTopicMetrics_FiveMinuteRate{name="BytesInPerSec",topic="__consumer_offsets",} 16.94886650744339 * will produce: * name=kafka_server_BrokerTopicMetrics_FiveMinuteRate * value=16.94886650744339 * labels={name="BytesInPerSec", topic="__consumer_offsets"}", */ private static final Pattern PATTERN = Pattern.compile( "(?^\\w+)([ \t]*\\{*(?.*)}*)[ \\t]+(?[\\d]+\\.?[\\d]+)?"); static Optional parse(String s) { Matcher matcher = PATTERN.matcher(s); if (matcher.matches()) { String value = matcher.group("value"); String metricName = matcher.group("metricName"); if (metricName == null || !NumberUtils.isCreatable(value)) { return Optional.empty(); } var labels = Arrays.stream(matcher.group("properties").split(",")) .filter(str -> !"".equals(str)) .map(str -> str.split("=")) .filter(spit -> spit.length == 2) .collect(Collectors.toUnmodifiableMap( str -> str[0].trim(), str -> str[1].trim().replace("\"", ""))); return Optional.of(RawMetric.create(metricName, labels, new BigDecimal(value))); } return Optional.empty(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetriever.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.MetricsConfig; import com.provectus.kafka.ui.util.WebClientConfigurator; import java.util.Arrays; import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.Node; import org.springframework.stereotype.Service; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Service @Slf4j class PrometheusMetricsRetriever implements MetricsRetriever { private static final String METRICS_ENDPOINT_PATH = "/metrics"; private static final int DEFAULT_EXPORTER_PORT = 11001; @Override public Flux retrieve(KafkaCluster c, Node node) { log.debug("Retrieving metrics from prometheus exporter: {}:{}", node.host(), c.getMetricsConfig().getPort()); MetricsConfig metricsConfig = c.getMetricsConfig(); var webClient = new WebClientConfigurator() .configureBufferSize(DataSize.ofMegabytes(20)) .configureBasicAuth(metricsConfig.getUsername(), metricsConfig.getPassword()) .configureSsl( c.getOriginalProperties().getSsl(), new ClustersProperties.KeystoreConfig( metricsConfig.getKeystoreLocation(), metricsConfig.getKeystorePassword())) .build(); return retrieve(webClient, node.host(), c.getMetricsConfig()); } @VisibleForTesting Flux retrieve(WebClient webClient, String host, MetricsConfig metricsConfig) { int port = Optional.ofNullable(metricsConfig.getPort()).orElse(DEFAULT_EXPORTER_PORT); boolean sslEnabled = metricsConfig.isSsl() || metricsConfig.getKeystoreLocation() != null; var request = webClient.get() .uri(UriComponentsBuilder.newInstance() .scheme(sslEnabled ? "https" : "http") .host(host) .port(port) .path(METRICS_ENDPOINT_PATH).build().toUri()); WebClient.ResponseSpec responseSpec = request.retrieve(); return responseSpec.bodyToMono(String.class) .doOnError(e -> log.error("Error while getting metrics from {}", host, e)) .onErrorResume(th -> Mono.empty()) .flatMapMany(body -> Flux.fromStream( Arrays.stream(body.split("\\n")) .filter(str -> !Strings.isNullOrEmpty(str) && !str.startsWith("#")) // skipping comments strings .map(PrometheusEndpointMetricsParser::parse) .filter(Optional::isPresent) .map(Optional::get) ) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/RawMetric.java ================================================ package com.provectus.kafka.ui.service.metrics; import java.math.BigDecimal; import java.util.Map; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.ToString; public interface RawMetric { String name(); Map labels(); BigDecimal value(); // Key, that can be used for metrics reductions default Object identityKey() { return name() + "_" + labels(); } RawMetric copyWithValue(BigDecimal newValue); //-------------------------------------------------- static RawMetric create(String name, Map labels, BigDecimal value) { return new SimpleMetric(name, labels, value); } @AllArgsConstructor @EqualsAndHashCode @ToString class SimpleMetric implements RawMetric { private final String name; private final Map labels; private final BigDecimal value; @Override public String name() { return name; } @Override public Map labels() { return labels; } @Override public BigDecimal value() { return value; } @Override public RawMetric copyWithValue(BigDecimal newValue) { return new SimpleMetric(name, labels, newValue); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/metrics/WellKnownMetrics.java ================================================ package com.provectus.kafka.ui.service.metrics; import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; import static org.apache.commons.lang3.StringUtils.endsWithIgnoreCase; import com.provectus.kafka.ui.model.Metrics; import java.math.BigDecimal; import java.util.HashMap; import java.util.Map; import org.apache.kafka.common.Node; class WellKnownMetrics { private static final String BROKER_TOPIC_METRICS = "BrokerTopicMetrics"; private static final String FIFTEEN_MINUTE_RATE = "FifteenMinuteRate"; // per broker final Map brokerBytesInFifteenMinuteRate = new HashMap<>(); final Map brokerBytesOutFifteenMinuteRate = new HashMap<>(); // per topic final Map bytesInFifteenMinuteRate = new HashMap<>(); final Map bytesOutFifteenMinuteRate = new HashMap<>(); void populate(Node node, RawMetric rawMetric) { updateBrokerIOrates(node, rawMetric); updateTopicsIOrates(rawMetric); } void apply(Metrics.MetricsBuilder metricsBuilder) { metricsBuilder.topicBytesInPerSec(bytesInFifteenMinuteRate); metricsBuilder.topicBytesOutPerSec(bytesOutFifteenMinuteRate); metricsBuilder.brokerBytesInPerSec(brokerBytesInFifteenMinuteRate); metricsBuilder.brokerBytesOutPerSec(brokerBytesOutFifteenMinuteRate); } private void updateBrokerIOrates(Node node, RawMetric rawMetric) { String name = rawMetric.name(); if (!brokerBytesInFifteenMinuteRate.containsKey(node.id()) && rawMetric.labels().size() == 1 && "BytesInPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) && containsIgnoreCase(name, BROKER_TOPIC_METRICS) && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { brokerBytesInFifteenMinuteRate.put(node.id(), rawMetric.value()); } if (!brokerBytesOutFifteenMinuteRate.containsKey(node.id()) && rawMetric.labels().size() == 1 && "BytesOutPerSec".equalsIgnoreCase(rawMetric.labels().get("name")) && containsIgnoreCase(name, BROKER_TOPIC_METRICS) && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { brokerBytesOutFifteenMinuteRate.put(node.id(), rawMetric.value()); } } private void updateTopicsIOrates(RawMetric rawMetric) { String name = rawMetric.name(); String topic = rawMetric.labels().get("topic"); if (topic != null && containsIgnoreCase(name, BROKER_TOPIC_METRICS) && endsWithIgnoreCase(name, FIFTEEN_MINUTE_RATE)) { String nameProperty = rawMetric.labels().get("name"); if ("BytesInPerSec".equalsIgnoreCase(nameProperty)) { bytesInFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); } else if ("BytesOutPerSec".equalsIgnoreCase(nameProperty)) { bytesOutFifteenMinuteRate.compute(topic, (k, v) -> v == null ? rawMetric.value() : v.add(rawMetric.value())); } } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AbstractProviderCondition.java ================================================ package com.provectus.kafka.ui.service.rbac; import com.provectus.kafka.ui.config.auth.OAuthProperties; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.context.properties.bind.Bindable; import org.springframework.boot.context.properties.bind.Binder; import org.springframework.core.env.Environment; public abstract class AbstractProviderCondition { private static final Bindable> OAUTH2_PROPERTIES = Bindable .mapOf(String.class, OAuthProperties.OAuth2Provider.class); protected Set getRegisteredProvidersTypes(final Environment env) { final Map properties = Binder.get(env) .bind("auth.oauth2.client", OAUTH2_PROPERTIES) .orElse(Map.of()); return properties.values().stream() .map(OAuthProperties.OAuth2Provider::getCustomParams) .filter(Objects::nonNull) .filter(Predicate.not(Map::isEmpty)) .map(params -> params.get("type")) .filter(Objects::nonNull) .filter(StringUtils::isNotEmpty) .collect(Collectors.toSet()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java ================================================ package com.provectus.kafka.ui.service.rbac; import static com.provectus.kafka.ui.model.rbac.Resource.APPLICATIONCONFIG; import com.provectus.kafka.ui.config.auth.AuthenticatedUser; import com.provectus.kafka.ui.config.auth.RbacUser; import com.provectus.kafka.ui.config.auth.RoleBasedAccessControlProperties; import com.provectus.kafka.ui.model.ClusterDTO; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.Permission; import com.provectus.kafka.ui.model.rbac.Resource; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.Subject; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import com.provectus.kafka.ui.service.rbac.extractor.CognitoAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GithubAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.GoogleAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.OauthAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.ProviderAuthorityExtractor; import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.core.env.Environment; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.context.ReactiveSecurityContextHolder; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.oauth2.client.registration.InMemoryReactiveClientRegistrationRepository; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import reactor.core.publisher.Mono; @Service @RequiredArgsConstructor @EnableConfigurationProperties(RoleBasedAccessControlProperties.class) @Slf4j public class AccessControlService { private static final String ACCESS_DENIED = "Access denied"; private static final String ACTIONS_ARE_EMPTY = "actions are empty"; @Nullable private final InMemoryReactiveClientRegistrationRepository clientRegistrationRepository; private final RoleBasedAccessControlProperties properties; private final Environment environment; private boolean rbacEnabled = false; private Set oauthExtractors = Collections.emptySet(); @PostConstruct public void init() { if (CollectionUtils.isEmpty(properties.getRoles())) { log.trace("No roles provided, disabling RBAC"); return; } rbacEnabled = true; this.oauthExtractors = properties.getRoles() .stream() .map(role -> role.getSubjects() .stream() .map(Subject::getProvider) .distinct() .map(provider -> switch (provider) { case OAUTH_COGNITO -> new CognitoAuthorityExtractor(); case OAUTH_GOOGLE -> new GoogleAuthorityExtractor(); case OAUTH_GITHUB -> new GithubAuthorityExtractor(); case OAUTH -> new OauthAuthorityExtractor(); default -> null; }) .filter(Objects::nonNull) .collect(Collectors.toSet())) .flatMap(Set::stream) .collect(Collectors.toSet()); if (!properties.getRoles().isEmpty() && "oauth2".equalsIgnoreCase(environment.getProperty("auth.type")) && (clientRegistrationRepository == null || !clientRegistrationRepository.iterator().hasNext())) { log.error("Roles are configured but no authentication methods are present. Authentication might fail."); } } public Mono validateAccess(AccessContext context) { if (!rbacEnabled) { return Mono.empty(); } if (CollectionUtils.isNotEmpty(context.getApplicationConfigActions())) { return getUser() .doOnNext(user -> { boolean accessGranted = isApplicationConfigAccessible(context, user); if (!accessGranted) { throw new AccessDeniedException(ACCESS_DENIED); } }).then(); } return getUser() .doOnNext(user -> { boolean accessGranted = isApplicationConfigAccessible(context, user) && isClusterAccessible(context, user) && isClusterConfigAccessible(context, user) && isTopicAccessible(context, user) && isConsumerGroupAccessible(context, user) && isConnectAccessible(context, user) && isConnectorAccessible(context, user) // TODO connector selectors && isSchemaAccessible(context, user) && isKsqlAccessible(context, user) && isAclAccessible(context, user) && isAuditAccessible(context, user); if (!accessGranted) { throw new AccessDeniedException(ACCESS_DENIED); } }) .then(); } public Mono getUser() { return ReactiveSecurityContextHolder.getContext() .map(SecurityContext::getAuthentication) .filter(authentication -> authentication.getPrincipal() instanceof RbacUser) .map(authentication -> ((RbacUser) authentication.getPrincipal())) .map(user -> new AuthenticatedUser(user.name(), user.groups())); } public boolean isApplicationConfigAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (CollectionUtils.isEmpty(context.getApplicationConfigActions())) { return true; } Set requiredActions = context.getApplicationConfigActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(APPLICATIONCONFIG, null, user, context, requiredActions); } private boolean isClusterAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } Assert.isTrue(StringUtils.isNotEmpty(context.getCluster()), "cluster value is empty"); return properties.getRoles() .stream() .filter(filterRole(user)) .anyMatch(filterCluster(context.getCluster())); } public Mono isClusterAccessible(ClusterDTO cluster) { if (!rbacEnabled) { return Mono.just(true); } AccessContext accessContext = AccessContext .builder() .cluster(cluster.getName()) .build(); return getUser().map(u -> isClusterAccessible(accessContext, u)); } public boolean isClusterConfigAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (CollectionUtils.isEmpty(context.getClusterConfigActions())) { return true; } Assert.isTrue(StringUtils.isNotEmpty(context.getCluster()), "cluster value is empty"); Set requiredActions = context.getClusterConfigActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.CLUSTERCONFIG, context.getCluster(), user, context, requiredActions); } public boolean isTopicAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getTopic() == null && context.getTopicActions().isEmpty()) { return true; } Assert.isTrue(!context.getTopicActions().isEmpty(), ACTIONS_ARE_EMPTY); Set requiredActions = context.getTopicActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.TOPIC, context.getTopic(), user, context, requiredActions); } public Mono> filterViewableTopics(List topics, String clusterName) { if (!rbacEnabled) { return Mono.just(topics); } return getUser() .map(user -> topics.stream() .filter(topic -> { var accessContext = AccessContext .builder() .cluster(clusterName) .topic(topic.getName()) .topicActions(TopicAction.VIEW) .build(); return isTopicAccessible(accessContext, user); } ).toList()); } private boolean isConsumerGroupAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getConsumerGroup() == null && context.getConsumerGroupActions().isEmpty()) { return true; } Assert.isTrue(!context.getConsumerGroupActions().isEmpty(), ACTIONS_ARE_EMPTY); Set requiredActions = context.getConsumerGroupActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.CONSUMER, context.getConsumerGroup(), user, context, requiredActions); } public Mono isConsumerGroupAccessible(String groupId, String clusterName) { if (!rbacEnabled) { return Mono.just(true); } AccessContext accessContext = AccessContext .builder() .cluster(clusterName) .consumerGroup(groupId) .consumerGroupActions(ConsumerGroupAction.VIEW) .build(); return getUser().map(u -> isConsumerGroupAccessible(accessContext, u)); } public boolean isSchemaAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getSchema() == null && context.getSchemaActions().isEmpty()) { return true; } Assert.isTrue(!context.getSchemaActions().isEmpty(), ACTIONS_ARE_EMPTY); Set requiredActions = context.getSchemaActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.SCHEMA, context.getSchema(), user, context, requiredActions); } public Mono isSchemaAccessible(String schema, String clusterName) { if (!rbacEnabled) { return Mono.just(true); } AccessContext accessContext = AccessContext .builder() .cluster(clusterName) .schema(schema) .schemaActions(SchemaAction.VIEW) .build(); return getUser().map(u -> isSchemaAccessible(accessContext, u)); } public boolean isConnectAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getConnect() == null && context.getConnectActions().isEmpty()) { return true; } Assert.isTrue(!context.getConnectActions().isEmpty(), ACTIONS_ARE_EMPTY); Set requiredActions = context.getConnectActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.CONNECT, context.getConnect(), user, context, requiredActions); } public Mono isConnectAccessible(ConnectDTO dto, String clusterName) { if (!rbacEnabled) { return Mono.just(true); } return isConnectAccessible(dto.getName(), clusterName); } public Mono isConnectAccessible(String connectName, String clusterName) { if (!rbacEnabled) { return Mono.just(true); } AccessContext accessContext = AccessContext .builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .build(); return getUser().map(u -> isConnectAccessible(accessContext, u)); } public boolean isConnectorAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } return isConnectAccessible(context, user); } public Mono isConnectorAccessible(String connectName, String connectorName, String clusterName) { if (!rbacEnabled) { return Mono.just(true); } AccessContext accessContext = AccessContext .builder() .cluster(clusterName) .connect(connectName) .connectActions(ConnectAction.VIEW) .connector(connectorName) .build(); return getUser().map(u -> isConnectorAccessible(accessContext, u)); } private boolean isKsqlAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getKsqlActions().isEmpty()) { return true; } Set requiredActions = context.getKsqlActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.KSQL, null, user, context, requiredActions); } private boolean isAclAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getAclActions().isEmpty()) { return true; } Set requiredActions = context.getAclActions() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.ACL, null, user, context, requiredActions); } private boolean isAuditAccessible(AccessContext context, AuthenticatedUser user) { if (!rbacEnabled) { return true; } if (context.getAuditAction().isEmpty()) { return true; } Set requiredActions = context.getAuditAction() .stream() .map(a -> a.toString().toUpperCase()) .collect(Collectors.toSet()); return isAccessible(Resource.AUDIT, null, user, context, requiredActions); } public Set getOauthExtractors() { return oauthExtractors; } public List getRoles() { if (!rbacEnabled) { return Collections.emptyList(); } return Collections.unmodifiableList(properties.getRoles()); } private boolean isAccessible(Resource resource, @Nullable String resourceValue, AuthenticatedUser user, AccessContext context, Set requiredActions) { Set grantedActions = properties.getRoles() .stream() .filter(filterRole(user)) .filter(filterCluster(resource, context.getCluster())) .flatMap(grantedRole -> grantedRole.getPermissions().stream()) .filter(filterResource(resource)) .filter(filterResourceValue(resourceValue)) .flatMap(grantedPermission -> grantedPermission.getActions().stream()) .map(String::toUpperCase) .collect(Collectors.toSet()); return grantedActions.containsAll(requiredActions); } private Predicate filterRole(AuthenticatedUser user) { return role -> user.groups().contains(role.getName()); } private Predicate filterCluster(String cluster) { return grantedRole -> grantedRole.getClusters() .stream() .anyMatch(cluster::equalsIgnoreCase); } private Predicate filterCluster(Resource resource, String cluster) { if (resource == APPLICATIONCONFIG) { return role -> true; } return filterCluster(cluster); } private Predicate filterResource(Resource resource) { return grantedPermission -> resource == grantedPermission.getResource(); } private Predicate filterResourceValue(@Nullable String resourceValue) { if (resourceValue == null) { return grantedPermission -> true; } return grantedPermission -> { Pattern valuePattern = grantedPermission.getCompiledValuePattern(); if (valuePattern == null) { return true; } return valuePattern.matcher(resourceValue).matches(); }; } public boolean isRbacEnabled() { return rbacEnabled; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import static com.provectus.kafka.ui.model.rbac.provider.Provider.Name.COGNITO; import com.google.common.collect.Sets; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import reactor.core.publisher.Mono; @Slf4j public class CognitoAuthorityExtractor implements ProviderAuthorityExtractor { private static final String COGNITO_GROUPS_ATTRIBUTE_NAME = "cognito:groups"; @Override public boolean isApplicable(String provider, Map customParams) { return COGNITO.equalsIgnoreCase(provider) || COGNITO.equalsIgnoreCase(customParams.get(TYPE)); } @Override public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { log.debug("Extracting cognito user authorities"); DefaultOAuth2User principal; try { principal = (DefaultOAuth2User) value; } catch (ClassCastException e) { log.error("Can't cast value to DefaultOAuth2User", e); throw new RuntimeException(); } Set groupsByUsername = acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_COGNITO)) .filter(s -> s.getType().equals("user")) .anyMatch(s -> s.getValue().equals(principal.getName()))) .map(Role::getName) .collect(Collectors.toSet()); List groups = principal.getAttribute(COGNITO_GROUPS_ATTRIBUTE_NAME); if (groups == null) { log.debug("Cognito groups param is not present"); return Mono.just(groupsByUsername); } Set groupsByGroups = acs.getRoles() .stream() .filter(role -> role.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_COGNITO)) .filter(s -> s.getType().equals("group")) .anyMatch(subject -> groups .stream() .anyMatch(cognitoGroup -> cognitoGroup.equals(subject.getValue())) )) .map(Role::getName) .collect(Collectors.toSet()); return Mono.just(Sets.union(groupsByUsername, groupsByGroups)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/GithubAuthorityExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import static com.provectus.kafka.ui.model.rbac.provider.Provider.Name.GITHUB; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpHeaders; import org.springframework.security.config.oauth2.client.CommonOAuth2Provider; import org.springframework.security.oauth2.client.userinfo.OAuth2UserRequest; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; @Slf4j public class GithubAuthorityExtractor implements ProviderAuthorityExtractor { private static final String ORGANIZATION_ATTRIBUTE_NAME = "organizations_url"; private static final String USERNAME_ATTRIBUTE_NAME = "login"; private static final String ORGANIZATION_NAME = "login"; private static final String ORGANIZATION = "organization"; private static final String TEAM_NAME = "slug"; private static final String GITHUB_ACCEPT_HEADER = "application/vnd.github+json"; private static final String DUMMY = "dummy"; // The number of results (max 100) per page of list organizations for authenticated user. private static final Integer ORGANIZATIONS_PER_PAGE = 100; @Override public boolean isApplicable(String provider, Map customParams) { return GITHUB.equalsIgnoreCase(provider) || GITHUB.equalsIgnoreCase(customParams.get(TYPE)); } @Override public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { DefaultOAuth2User principal; try { principal = (DefaultOAuth2User) value; } catch (ClassCastException e) { log.error("Can't cast value to DefaultOAuth2User", e); throw new RuntimeException(); } Set rolesByUsername = new HashSet<>(); String username = principal.getAttribute(USERNAME_ATTRIBUTE_NAME); if (username == null) { log.debug("Github username param is not present"); } else { acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GITHUB)) .filter(s -> s.getType().equals("user")) .anyMatch(s -> s.getValue().equals(username))) .map(Role::getName) .forEach(rolesByUsername::add); } OAuth2UserRequest req = (OAuth2UserRequest) additionalParams.get("request"); String infoEndpoint = req.getClientRegistration().getProviderDetails().getUserInfoEndpoint().getUri(); if (infoEndpoint == null) { infoEndpoint = CommonOAuth2Provider.GITHUB .getBuilder(DUMMY) .clientId(DUMMY) .build() .getProviderDetails() .getUserInfoEndpoint() .getUri(); } var webClient = WebClient.create(infoEndpoint); Mono> rolesByOrganization = getOrganizationRoles(principal, additionalParams, acs, webClient); Mono> rolesByTeams = getTeamRoles(webClient, additionalParams, acs); return Mono.zip(rolesByOrganization, rolesByTeams) .map((t) -> Stream.of(t.getT1(), t.getT2(), rolesByUsername) .flatMap(Collection::stream) .collect(Collectors.toSet())); } private Mono> getOrganizationRoles(DefaultOAuth2User principal, Map additionalParams, AccessControlService acs, WebClient webClient) { String organization = principal.getAttribute(ORGANIZATION_ATTRIBUTE_NAME); if (organization == null) { log.debug("Github organization param is not present"); return Mono.just(Collections.emptySet()); } final Mono>> userOrganizations = webClient .get() .uri(uriBuilder -> uriBuilder.path("/orgs") .queryParam("per_page", ORGANIZATIONS_PER_PAGE) .build()) .headers(headers -> { headers.set(HttpHeaders.ACCEPT, GITHUB_ACCEPT_HEADER); OAuth2UserRequest request = (OAuth2UserRequest) additionalParams.get("request"); headers.setBearerAuth(request.getAccessToken().getTokenValue()); }) .retrieve() //@formatter:off .bodyToMono(new ParameterizedTypeReference<>() {}); //@formatter:on return userOrganizations .map(orgsMap -> acs.getRoles() .stream() .filter(role -> role.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GITHUB)) .filter(s -> s.getType().equals(ORGANIZATION)) .anyMatch(subject -> orgsMap.stream() .map(org -> org.get(ORGANIZATION_NAME).toString()) .anyMatch(orgName -> orgName.equalsIgnoreCase(subject.getValue())) )) .map(Role::getName) .collect(Collectors.toSet())); } @SuppressWarnings("unchecked") private Mono> getTeamRoles(WebClient webClient, Map additionalParams, AccessControlService acs) { var requestedTeams = acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GITHUB)) .anyMatch(s -> s.getType().equals("team"))) .collect(Collectors.toSet()); if (requestedTeams.isEmpty()) { log.debug("No roles with github teams found, skipping"); return Mono.just(Collections.emptySet()); } final Mono>> rawTeams = webClient .get() .uri(uriBuilder -> uriBuilder.path("/teams") .queryParam("per_page", ORGANIZATIONS_PER_PAGE) .build()) .headers(headers -> { headers.set(HttpHeaders.ACCEPT, GITHUB_ACCEPT_HEADER); OAuth2UserRequest request = (OAuth2UserRequest) additionalParams.get("request"); headers.setBearerAuth(request.getAccessToken().getTokenValue()); }) .retrieve() //@formatter:off .bodyToMono(new ParameterizedTypeReference<>() {}); //@formatter:on final Mono> mappedTeams = rawTeams .map(teams -> teams.stream() .map(teamInfo -> { var name = teamInfo.get(TEAM_NAME); var orgInfo = (Map) teamInfo.get(ORGANIZATION); var orgName = orgInfo.get(ORGANIZATION_NAME); return orgName + "/" + name; }) .map(Object::toString) .collect(Collectors.toList()) ); return mappedTeams .map(teams -> acs.getRoles() .stream() .filter(role -> role.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GITHUB)) .filter(s -> s.getType().equals("team")) .anyMatch(subject -> teams.stream() .anyMatch(teamName -> teamName.equalsIgnoreCase(subject.getValue())) )) .map(Role::getName) .collect(Collectors.toSet())); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/GoogleAuthorityExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import static com.provectus.kafka.ui.model.rbac.provider.Provider.Name.GOOGLE; import com.google.common.collect.Sets; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import reactor.core.publisher.Mono; @Slf4j public class GoogleAuthorityExtractor implements ProviderAuthorityExtractor { private static final String GOOGLE_DOMAIN_ATTRIBUTE_NAME = "hd"; public static final String EMAIL_ATTRIBUTE_NAME = "email"; @Override public boolean isApplicable(String provider, Map customParams) { return GOOGLE.equalsIgnoreCase(provider) || GOOGLE.equalsIgnoreCase(customParams.get(TYPE)); } @Override public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { log.debug("Extracting google user authorities"); DefaultOAuth2User principal; try { principal = (DefaultOAuth2User) value; } catch (ClassCastException e) { log.error("Can't cast value to DefaultOAuth2User", e); throw new RuntimeException(); } Set groupsByUsername = acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GOOGLE)) .filter(s -> s.getType().equals("user")) .anyMatch(s -> s.getValue().equals(principal.getAttribute(EMAIL_ATTRIBUTE_NAME)))) .map(Role::getName) .collect(Collectors.toSet()); String domain = principal.getAttribute(GOOGLE_DOMAIN_ATTRIBUTE_NAME); if (domain == null) { log.debug("Google domain param is not present"); return Mono.just(groupsByUsername); } Set groupsByDomain = acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH_GOOGLE)) .filter(s -> s.getType().equals("domain")) .anyMatch(s -> s.getValue().equals(domain))) .map(Role::getName) .collect(Collectors.toSet()); return Mono.just(Sets.union(groupsByUsername, groupsByDomain)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/OauthAuthorityExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import static com.provectus.kafka.ui.model.rbac.provider.Provider.Name.OAUTH; import com.google.common.collect.Sets; import com.provectus.kafka.ui.config.auth.OAuthProperties; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import org.springframework.util.Assert; import reactor.core.publisher.Mono; @Slf4j public class OauthAuthorityExtractor implements ProviderAuthorityExtractor { public static final String ROLES_FIELD_PARAM_NAME = "roles-field"; @Override public boolean isApplicable(String provider, Map customParams) { var containsRolesFieldNameParam = customParams.containsKey(ROLES_FIELD_PARAM_NAME); if (!containsRolesFieldNameParam) { log.debug("Provider [{}] doesn't contain a roles field param name, mapping won't be performed", provider); return false; } return OAUTH.equalsIgnoreCase(provider) || OAUTH.equalsIgnoreCase(customParams.get(TYPE)); } @Override public Mono> extract(AccessControlService acs, Object value, Map additionalParams) { log.trace("Extracting OAuth2 user authorities"); DefaultOAuth2User principal; try { principal = (DefaultOAuth2User) value; } catch (ClassCastException e) { log.error("Can't cast value to DefaultOAuth2User", e); throw new RuntimeException(); } var provider = (OAuthProperties.OAuth2Provider) additionalParams.get("provider"); Assert.notNull(provider, "provider is null"); var rolesFieldName = provider.getCustomParams().get(ROLES_FIELD_PARAM_NAME); Set rolesByUsername = acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH)) .filter(s -> s.getType().equals("user")) .anyMatch(s -> s.getValue().equals(principal.getName()))) .map(Role::getName) .collect(Collectors.toSet()); Set rolesByRolesField = acs.getRoles() .stream() .filter(role -> role.getSubjects() .stream() .filter(s -> s.getProvider().equals(Provider.OAUTH)) .filter(s -> s.getType().equals("role")) .anyMatch(subject -> { var roleName = subject.getValue(); var principalRoles = convertRoles(principal.getAttribute(rolesFieldName)); var roleMatched = principalRoles.contains(roleName); if (roleMatched) { log.debug("Assigning role [{}] to user [{}]", roleName, principal.getName()); } else { log.trace("Role [{}] not found in user [{}] roles", roleName, principal.getName()); } return roleMatched; }) ) .map(Role::getName) .collect(Collectors.toSet()); return Mono.just(Sets.union(rolesByUsername, rolesByRolesField)); } @SuppressWarnings("unchecked") private Collection convertRoles(Object roles) { if (roles == null) { log.debug("Param missing from attributes, skipping"); return Collections.emptySet(); } if ((roles instanceof List) || (roles instanceof Set)) { log.trace("The field is either a set or a list, returning as is"); return (Collection) roles; } if (!(roles instanceof String)) { log.debug("The field is not a string, skipping"); return Collections.emptySet(); } log.trace("Trying to deserialize the field value [{}] as a string", roles); return Arrays.stream(((String) roles).split(",")) .collect(Collectors.toSet()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/ProviderAuthorityExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.Map; import java.util.Set; import reactor.core.publisher.Mono; public interface ProviderAuthorityExtractor { String TYPE = "type"; boolean isApplicable(String provider, Map customParams); Mono> extract(AccessControlService acs, Object value, Map additionalParams); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/RbacLdapAuthoritiesExtractor.java ================================================ package com.provectus.kafka.ui.service.rbac.extractor; import com.provectus.kafka.ui.config.auth.LdapProperties; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationContext; import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopulator; import org.springframework.util.Assert; @Slf4j public class RbacLdapAuthoritiesExtractor extends DefaultLdapAuthoritiesPopulator { private final AccessControlService acs; private final LdapProperties props; public RbacLdapAuthoritiesExtractor(ApplicationContext context, BaseLdapPathContextSource contextSource, String groupFilterSearchBase) { super(contextSource, groupFilterSearchBase); this.acs = context.getBean(AccessControlService.class); this.props = context.getBean(LdapProperties.class); } @Override protected Set getAdditionalRoles(DirContextOperations user, String username) { var ldapGroups = getRoles(user.getNameInNamespace(), username); return acs.getRoles() .stream() .filter(r -> r.getSubjects() .stream() .filter(subject -> subject.getProvider().equals(Provider.LDAP)) .filter(subject -> subject.getType().equals("group")) .anyMatch(subject -> ldapGroups.contains(subject.getValue())) ) .map(Role::getName) .peek(role -> log.trace("Mapped role [{}] for user [{}]", role, username)) .map(SimpleGrantedAuthority::new) .collect(Collectors.toSet()); } private Set getRoles(String userDn, String username) { var groupSearchBase = props.getGroupFilterSearchBase(); Assert.notNull(groupSearchBase, "groupSearchBase is empty"); var groupRoleAttribute = props.getGroupRoleAttribute(); if (groupRoleAttribute == null) { groupRoleAttribute = "cn"; } log.trace( "Searching for roles for user [{}] with DN [{}], groupRoleAttribute [{}] and filter [{}] in search base [{}]", username, userDn, groupRoleAttribute, getGroupSearchFilter(), groupSearchBase); var ldapTemplate = getLdapTemplate(); ldapTemplate.setIgnoreNameNotFoundException(true); Set>> userRoles = ldapTemplate.searchForMultipleAttributeValues( groupSearchBase, getGroupSearchFilter(), new String[] {userDn, username}, new String[] {groupRoleAttribute}); return userRoles.stream() .map(record -> record.get(getGroupRoleAttribute()).get(0)) .peek(group -> log.trace("Found LDAP group [{}] for user [{}]", group, username)) .collect(Collectors.toSet()); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ApplicationMetrics.java ================================================ package com.provectus.kafka.ui.util; import static lombok.AccessLevel.PRIVATE; import com.google.common.annotations.VisibleForTesting; import com.provectus.kafka.ui.emitter.PolledRecords; import com.provectus.kafka.ui.model.KafkaCluster; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.util.concurrent.atomic.AtomicInteger; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor(access = PRIVATE) public class ApplicationMetrics { // kafka-ui specific metrics prefix. Added to make it easier to distinguish kui metrics from // other metrics, exposed by spring boot (like http stats, jvm, etc.) private static final String COMMON_PREFIX = "kui_"; private final String clusterName; private final MeterRegistry registry; public static ApplicationMetrics forCluster(KafkaCluster cluster) { return new ApplicationMetrics(cluster.getName(), Metrics.globalRegistry); } @VisibleForTesting public static ApplicationMetrics noop() { return new ApplicationMetrics("noop", new SimpleMeterRegistry()); } public void meterPolledRecords(String topic, PolledRecords polled, boolean throttled) { pollTimer(topic).record(polled.elapsed()); polledRecords(topic).increment(polled.count()); polledBytes(topic).record(polled.bytes()); if (throttled) { pollThrottlingActivations().increment(); } } private Counter polledRecords(String topic) { return Counter.builder(COMMON_PREFIX + "topic_records_polled") .description("Number of records polled from topic") .tag("cluster", clusterName) .tag("topic", topic) .register(registry); } private DistributionSummary polledBytes(String topic) { return DistributionSummary.builder(COMMON_PREFIX + "topic_polled_bytes") .description("Bytes polled from kafka topic") .tag("cluster", clusterName) .tag("topic", topic) .register(registry); } private Timer pollTimer(String topic) { return Timer.builder(COMMON_PREFIX + "topic_poll_time") .description("Time spend in polling for topic") .tag("cluster", clusterName) .tag("topic", topic) .register(registry); } private Counter pollThrottlingActivations() { return Counter.builder(COMMON_PREFIX + "poll_throttling_activations") .description("Number of poll throttling activations") .tag("cluster", clusterName) .register(registry); } public AtomicInteger activeConsumers() { var count = new AtomicInteger(); Gauge.builder(COMMON_PREFIX + "active_consumers", () -> count) .description("Number of active consumers") .tag("cluster", clusterName) .register(registry); return count; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ApplicationRestarter.java ================================================ package com.provectus.kafka.ui.util; import com.provectus.kafka.ui.KafkaUiApplication; import java.io.Closeable; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.event.ApplicationStartedEvent; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; @Slf4j @Component public class ApplicationRestarter implements ApplicationListener { private String[] applicationArgs; private ApplicationContext applicationContext; @Override public void onApplicationEvent(ApplicationStartedEvent event) { this.applicationArgs = event.getArgs(); this.applicationContext = event.getApplicationContext(); } public void requestRestart() { log.info("Restarting application"); Thread thread = new Thread(() -> { closeApplicationContext(applicationContext); KafkaUiApplication.startApplication(applicationArgs); }); thread.setName("restartedMain-" + System.currentTimeMillis()); thread.setDaemon(false); thread.start(); } private void closeApplicationContext(ApplicationContext context) { while (context instanceof Closeable) { try { ((Closeable) context).close(); } catch (Exception e) { log.warn("Error stopping application before restart", e); throw new RuntimeException(e); } context = context.getParent(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/DynamicConfigOperations.java ================================================ package com.provectus.kafka.ui.util; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.config.WebclientProperties; import com.provectus.kafka.ui.config.auth.OAuthProperties; import com.provectus.kafka.ui.config.auth.RoleBasedAccessControlProperties; import com.provectus.kafka.ui.exception.FileUploadException; import com.provectus.kafka.ui.exception.ValidationException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.time.Instant; import java.util.Optional; import javax.annotation.Nullable; import lombok.Builder; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.boot.env.YamlPropertySourceLoader; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.env.CompositePropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.io.FileSystemResource; import org.springframework.http.codec.multipart.FilePart; import org.springframework.stereotype.Component; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.introspector.BeanAccess; import org.yaml.snakeyaml.introspector.Property; import org.yaml.snakeyaml.introspector.PropertyUtils; import org.yaml.snakeyaml.nodes.NodeTuple; import org.yaml.snakeyaml.nodes.Tag; import org.yaml.snakeyaml.representer.Representer; import reactor.core.publisher.Mono; @Slf4j @RequiredArgsConstructor @Component public class DynamicConfigOperations { static final String DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY = "dynamic.config.enabled"; static final String FILTERING_GROOVY_ENABLED_PROPERTY = "filtering.groovy.enabled"; static final String DYNAMIC_CONFIG_PATH_ENV_PROPERTY = "dynamic.config.path"; static final String DYNAMIC_CONFIG_PATH_ENV_PROPERTY_DEFAULT = "/etc/kafkaui/dynamic_config.yaml"; static final String CONFIG_RELATED_UPLOADS_DIR_PROPERTY = "config.related.uploads.dir"; static final String CONFIG_RELATED_UPLOADS_DIR_DEFAULT = "/etc/kafkaui/uploads"; public static ApplicationContextInitializer dynamicConfigPropertiesInitializer() { return appCtx -> new DynamicConfigOperations(appCtx) .loadDynamicPropertySource() .ifPresent(source -> appCtx.getEnvironment().getPropertySources().addFirst(source)); } private final ConfigurableApplicationContext ctx; public boolean dynamicConfigEnabled() { return "true".equalsIgnoreCase(ctx.getEnvironment().getProperty(DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY)); } public boolean filteringGroovyEnabled() { return "true".equalsIgnoreCase(ctx.getEnvironment().getProperty(FILTERING_GROOVY_ENABLED_PROPERTY)); } private Path dynamicConfigFilePath() { return Paths.get( Optional.ofNullable(ctx.getEnvironment().getProperty(DYNAMIC_CONFIG_PATH_ENV_PROPERTY)) .orElse(DYNAMIC_CONFIG_PATH_ENV_PROPERTY_DEFAULT) ); } @SneakyThrows public Optional> loadDynamicPropertySource() { if (dynamicConfigEnabled()) { Path configPath = dynamicConfigFilePath(); if (!Files.exists(configPath) || !Files.isReadable(configPath)) { log.warn("Dynamic config file {} doesnt exist or not readable", configPath); return Optional.empty(); } var propertySource = new CompositePropertySource("dynamicProperties"); new YamlPropertySourceLoader() .load("dynamicProperties", new FileSystemResource(configPath)) .forEach(propertySource::addPropertySource); log.info("Dynamic config loaded from {}", configPath); return Optional.of(propertySource); } return Optional.empty(); } public PropertiesStructure getCurrentProperties() { checkIfDynamicConfigEnabled(); return PropertiesStructure.builder() .kafka(getNullableBean(ClustersProperties.class)) .rbac(getNullableBean(RoleBasedAccessControlProperties.class)) .auth( PropertiesStructure.Auth.builder() .type(ctx.getEnvironment().getProperty("auth.type")) .oauth2(getNullableBean(OAuthProperties.class)) .build()) .webclient(getNullableBean(WebclientProperties.class)) .build(); } @Nullable private T getNullableBean(Class clazz) { try { return ctx.getBean(clazz); } catch (NoSuchBeanDefinitionException nsbde) { return null; } } public void persist(PropertiesStructure properties) { checkIfDynamicConfigEnabled(); properties.initAndValidate(); String yaml = serializeToYaml(properties); writeYamlToFile(yaml, dynamicConfigFilePath()); } public Mono uploadConfigRelatedFile(FilePart file) { checkIfDynamicConfigEnabled(); String targetDirStr = ctx.getEnvironment() .getProperty(CONFIG_RELATED_UPLOADS_DIR_PROPERTY, CONFIG_RELATED_UPLOADS_DIR_DEFAULT); Path targetDir = Path.of(targetDirStr); if (!Files.exists(targetDir)) { try { Files.createDirectories(targetDir); } catch (IOException e) { return Mono.error( new FileUploadException("Error creating directory for uploads %s".formatted(targetDir), e)); } } Path targetFilePath = targetDir.resolve(file.filename() + "-" + Instant.now().getEpochSecond()); log.info("Uploading config-related file {}", targetFilePath); if (Files.exists(targetFilePath)) { log.info("File {} already exists, it will be overwritten", targetFilePath); } return file.transferTo(targetFilePath) .thenReturn(targetFilePath) .doOnError(th -> log.error("Error uploading file {}", targetFilePath, th)) .onErrorMap(th -> new FileUploadException(targetFilePath, th)); } public void checkIfFilteringGroovyEnabled() { if (!filteringGroovyEnabled()) { throw new ValidationException( "Groovy filters is not allowed. " + "Set filtering.groovy.enabled property to 'true' to enabled it."); } } private void checkIfDynamicConfigEnabled() { if (!dynamicConfigEnabled()) { throw new ValidationException( "Dynamic config change is not allowed. " + "Set dynamic.config.enabled property to 'true' to enabled it."); } } @SneakyThrows private void writeYamlToFile(String yaml, Path path) { if (Files.isDirectory(path)) { throw new ValidationException("Dynamic file path is a directory, but should be a file path"); } if (!Files.exists(path.getParent())) { Files.createDirectories(path.getParent()); } if (Files.exists(path) && !Files.isWritable(path)) { throw new ValidationException("File already exists and is not writable"); } try { Files.writeString( path, yaml, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING // to override existing file ); } catch (IOException e) { throw new ValidationException("Error writing to " + path, e); } } private String serializeToYaml(PropertiesStructure props) { //representer, that skips fields with null values Representer representer = new Representer(new DumperOptions()) { @Override protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { if (propertyValue == null) { return null; // if value of property is null, ignore it. } else { return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); } } }; var propertyUtils = new PropertyUtils(); propertyUtils.setBeanAccess(BeanAccess.FIELD); representer.setPropertyUtils(propertyUtils); representer.addClassTag(PropertiesStructure.class, Tag.MAP); //to avoid adding class tag representer.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); //use indent instead of {} return new Yaml(representer).dump(props); } ///--------------------------------------------------------------------- @Data @Builder // field name should be in sync with @ConfigurationProperties annotation public static class PropertiesStructure { private ClustersProperties kafka; private RoleBasedAccessControlProperties rbac; private Auth auth; private WebclientProperties webclient; @Data @Builder public static class Auth { String type; OAuthProperties oauth2; } public void initAndValidate() { Optional.ofNullable(kafka) .ifPresent(ClustersProperties::validateAndSetDefaults); Optional.ofNullable(rbac) .ifPresent(RoleBasedAccessControlProperties::init); Optional.ofNullable(auth) .flatMap(a -> Optional.ofNullable(a.oauth2)) .ifPresent(OAuthProperties::init); Optional.ofNullable(webclient) .ifPresent(WebclientProperties::validate); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/EmptyRedirectStrategy.java ================================================ package com.provectus.kafka.ui.util; import java.net.URI; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.security.web.server.ServerRedirectStrategy; import org.springframework.util.Assert; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; public class EmptyRedirectStrategy implements ServerRedirectStrategy { private HttpStatus httpStatus = HttpStatus.FOUND; private boolean contextRelative = true; public Mono sendRedirect(ServerWebExchange exchange, URI location) { Assert.notNull(exchange, "exchange cannot be null"); Assert.notNull(location, "location cannot be null"); return Mono.fromRunnable(() -> { ServerHttpResponse response = exchange.getResponse(); response.setStatusCode(this.httpStatus); response.getHeaders().setLocation(createLocation(exchange, location)); }); } private URI createLocation(ServerWebExchange exchange, URI location) { if (!this.contextRelative) { return location; } String url = location.getPath().isEmpty() ? "/" : location.toASCIIString(); if (url.startsWith("/")) { String context = exchange.getRequest().getPath().contextPath().value(); return URI.create(context + url); } return location; } public void setHttpStatus(HttpStatus httpStatus) { Assert.notNull(httpStatus, "httpStatus cannot be null"); this.httpStatus = httpStatus; } public void setContextRelative(boolean contextRelative) { this.contextRelative = contextRelative; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/GithubReleaseInfo.java ================================================ package com.provectus.kafka.ui.util; import com.google.common.annotations.VisibleForTesting; import java.time.Duration; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; @Slf4j public class GithubReleaseInfo { private static final String GITHUB_LATEST_RELEASE_RETRIEVAL_URL = "https://api.github.com/repos/provectus/kafka-ui/releases/latest"; private static final Duration GITHUB_API_MAX_WAIT_TIME = Duration.ofSeconds(2); public record GithubReleaseDto(String html_url, String tag_name, String published_at) { static GithubReleaseDto empty() { return new GithubReleaseDto(null, null, null); } } private volatile GithubReleaseDto release = GithubReleaseDto.empty(); private final Mono refreshMono; public GithubReleaseInfo() { this(GITHUB_LATEST_RELEASE_RETRIEVAL_URL); } @VisibleForTesting GithubReleaseInfo(String url) { this.refreshMono = new WebClientConfigurator().build() .get() .uri(url) .exchangeToMono(resp -> resp.bodyToMono(GithubReleaseDto.class)) .timeout(GITHUB_API_MAX_WAIT_TIME) .doOnError(th -> log.trace("Error getting latest github release info", th)) .onErrorResume(th -> true, th -> Mono.just(GithubReleaseDto.empty())) .doOnNext(release -> this.release = release) .then(); } public GithubReleaseDto get() { return release; } public Mono refresh() { return refreshMono; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaServicesValidation.java ================================================ package com.provectus.kafka.ui.util; import static com.provectus.kafka.ui.config.ClustersProperties.TruststoreConfig; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.model.ApplicationPropertyValidationDTO; import com.provectus.kafka.ui.service.ReactiveAdminClient; import com.provectus.kafka.ui.service.ksql.KsqlApiClient; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import java.io.FileInputStream; import java.security.KeyStore; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.function.Supplier; import javax.annotation.Nullable; import javax.net.ssl.TrustManagerFactory; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.springframework.util.ResourceUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j public final class KafkaServicesValidation { private KafkaServicesValidation() { } private static Mono valid() { return Mono.just(new ApplicationPropertyValidationDTO().error(false)); } private static Mono invalid(String errorMsg) { return Mono.just(new ApplicationPropertyValidationDTO().error(true).errorMessage(errorMsg)); } private static Mono invalid(Throwable th) { return Mono.just(new ApplicationPropertyValidationDTO().error(true).errorMessage(th.getMessage())); } /** * Returns error msg, if any. */ public static Optional validateTruststore(TruststoreConfig truststoreConfig) { if (truststoreConfig.getTruststoreLocation() != null && truststoreConfig.getTruststorePassword() != null) { try (FileInputStream fileInputStream = new FileInputStream( (ResourceUtils.getFile(truststoreConfig.getTruststoreLocation())))) { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(fileInputStream, truststoreConfig.getTruststorePassword().toCharArray()); TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); trustManagerFactory.init(trustStore); } catch (Exception e) { return Optional.of(e.getMessage()); } } return Optional.empty(); } public static Mono validateClusterConnection(String bootstrapServers, Properties clusterProps, @Nullable TruststoreConfig ssl) { Properties properties = new Properties(); SslPropertiesUtil.addKafkaSslProperties(ssl, properties); properties.putAll(clusterProps); properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); // editing properties to make validation faster properties.put(AdminClientConfig.RETRIES_CONFIG, 1); properties.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, 5_000); properties.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 5_000); properties.put(AdminClientConfig.CLIENT_ID_CONFIG, "kui-admin-client-validation-" + System.currentTimeMillis()); AdminClient adminClient = null; try { adminClient = AdminClient.create(properties); } catch (Exception e) { log.error("Error creating admin client during validation", e); return invalid("Error while creating AdminClient. See logs for details."); } return Mono.just(adminClient) .then(ReactiveAdminClient.toMono(adminClient.listTopics().names())) .then(valid()) .doOnTerminate(adminClient::close) .onErrorResume(th -> { log.error("Error connecting to cluster", th); return KafkaServicesValidation.invalid("Error connecting to cluster. See logs for details."); }); } public static Mono validateSchemaRegistry( Supplier> clientSupplier) { ReactiveFailover client; try { client = clientSupplier.get(); } catch (Exception e) { log.error("Error creating Schema Registry client", e); return invalid("Error creating Schema Registry client: " + e.getMessage()); } return client .mono(KafkaSrClientApi::getGlobalCompatibilityLevel) .then(valid()) .onErrorResume(KafkaServicesValidation::invalid); } public static Mono validateConnect( Supplier> clientSupplier) { ReactiveFailover client; try { client = clientSupplier.get(); } catch (Exception e) { log.error("Error creating Connect client", e); return invalid("Error creating Connect client: " + e.getMessage()); } return client.flux(KafkaConnectClientApi::getConnectorPlugins) .collectList() .then(valid()) .onErrorResume(KafkaServicesValidation::invalid); } public static Mono validateKsql( Supplier> clientSupplier) { ReactiveFailover client; try { client = clientSupplier.get(); } catch (Exception e) { log.error("Error creating Ksql client", e); return invalid("Error creating Ksql client: " + e.getMessage()); } return client.flux(c -> c.execute("SHOW VARIABLES;", Map.of())) .collectList() .flatMap(ksqlResults -> Flux.fromIterable(ksqlResults) .filter(KsqlApiClient.KsqlResponseTable::isError) .flatMap(err -> invalid("Error response from ksql: " + err)) .next() .switchIfEmpty(valid()) ) .onErrorResume(KafkaServicesValidation::invalid); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/KafkaVersion.java ================================================ package com.provectus.kafka.ui.util; import java.util.Optional; public final class KafkaVersion { private KafkaVersion() { } public static Optional parse(String version) throws NumberFormatException { try { final String[] parts = version.split("\\."); if (parts.length > 2) { version = parts[0] + "." + parts[1]; } return Optional.of(Float.parseFloat(version.split("-")[0])); } catch (Exception e) { return Optional.empty(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ReactiveFailover.java ================================================ package com.provectus.kafka.ui.util; import com.google.common.base.Preconditions; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; public class ReactiveFailover { public static final Duration DEFAULT_RETRY_GRACE_PERIOD_MS = Duration.ofSeconds(5); public static final Predicate CONNECTION_REFUSED_EXCEPTION_FILTER = error -> error.getCause() instanceof IOException && error.getCause().getMessage().contains("Connection refused"); private final List> publishers; private int currentIndex = 0; private final Predicate failoverExceptionsPredicate; private final String noAvailablePublishersMsg; // creates single-publisher failover (basically for tests usage) public static ReactiveFailover createNoop(T publisher) { return create( List.of(publisher), th -> true, "publisher is not available", DEFAULT_RETRY_GRACE_PERIOD_MS ); } public static ReactiveFailover create(List publishers, Predicate failoverExeptionsPredicate, String noAvailablePublishersMsg, Duration retryGracePeriodMs) { return new ReactiveFailover<>( publishers.stream().map(p -> new PublisherHolder<>(() -> p, retryGracePeriodMs.toMillis())).toList(), failoverExeptionsPredicate, noAvailablePublishersMsg ); } public static ReactiveFailover create(List args, Function factory, Predicate failoverExeptionsPredicate, String noAvailablePublishersMsg, Duration retryGracePeriodMs) { return new ReactiveFailover<>( args.stream().map(arg -> new PublisherHolder<>(() -> factory.apply(arg), retryGracePeriodMs.toMillis())).toList(), failoverExeptionsPredicate, noAvailablePublishersMsg ); } private ReactiveFailover(List> publishers, Predicate failoverExceptionsPredicate, String noAvailablePublishersMsg) { Preconditions.checkArgument(!publishers.isEmpty()); this.publishers = publishers; this.failoverExceptionsPredicate = failoverExceptionsPredicate; this.noAvailablePublishersMsg = noAvailablePublishersMsg; } public Mono mono(Function> f) { List> candidates = getActivePublishers(); if (candidates.isEmpty()) { return Mono.error(() -> new IllegalStateException(noAvailablePublishersMsg)); } return mono(f, candidates); } private Mono mono(Function> f, List> candidates) { var publisher = candidates.get(0); return publisher.get() .flatMap(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); if (candidates.size() == 1) { return Mono.error(th); } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Mono.error(th); } return mono(f, newCandidates); }); } public Flux flux(Function> f) { List> candidates = getActivePublishers(); if (candidates.isEmpty()) { return Flux.error(() -> new IllegalStateException(noAvailablePublishersMsg)); } return flux(f, candidates); } private Flux flux(Function> f, List> candidates) { var publisher = candidates.get(0); return publisher.get() .flatMapMany(f) .onErrorResume(failoverExceptionsPredicate, th -> { publisher.markFailed(); if (candidates.size() == 1) { return Flux.error(th); } var newCandidates = candidates.stream().skip(1).filter(PublisherHolder::isActive).toList(); if (newCandidates.isEmpty()) { return Flux.error(th); } return flux(f, newCandidates); }); } /** * Returns list of active publishers, starting with latest active. */ private synchronized List> getActivePublishers() { var result = new ArrayList>(); for (int i = 0, j = currentIndex; i < publishers.size(); i++) { var publisher = publishers.get(j); if (publisher.isActive()) { result.add(publisher); } else if (currentIndex == j) { currentIndex = ++currentIndex == publishers.size() ? 0 : currentIndex; } j = ++j == publishers.size() ? 0 : j; } return result; } static class PublisherHolder { private final long retryGracePeriodMs; private final Supplier supplier; private final AtomicLong lastErrorTs = new AtomicLong(); private T publisherInstance; PublisherHolder(Supplier supplier, long retryGracePeriodMs) { this.supplier = supplier; this.retryGracePeriodMs = retryGracePeriodMs; } synchronized Mono get() { if (publisherInstance == null) { try { publisherInstance = supplier.get(); } catch (Throwable th) { return Mono.error(th); } } return Mono.just(publisherInstance); } void markFailed() { lastErrorTs.set(System.currentTimeMillis()); } boolean isActive() { return System.currentTimeMillis() - lastErrorTs.get() > retryGracePeriodMs; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/ResourceUtil.java ================================================ package com.provectus.kafka.ui.util; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.StandardCharsets; import org.springframework.core.io.Resource; import org.springframework.util.FileCopyUtils; public class ResourceUtil { private ResourceUtil() { } public static String readAsString(Resource resource) throws IOException { try (Reader reader = new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8)) { return FileCopyUtils.copyToString(reader); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/SslPropertiesUtil.java ================================================ package com.provectus.kafka.ui.util; import com.provectus.kafka.ui.config.ClustersProperties; import java.util.Properties; import javax.annotation.Nullable; import org.apache.kafka.common.config.SslConfigs; public final class SslPropertiesUtil { private SslPropertiesUtil() { } public static void addKafkaSslProperties(@Nullable ClustersProperties.TruststoreConfig truststoreConfig, Properties sink) { if (truststoreConfig != null && truststoreConfig.getTruststoreLocation() != null) { sink.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreConfig.getTruststoreLocation()); if (truststoreConfig.getTruststorePassword() != null) { sink.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, truststoreConfig.getTruststorePassword()); } } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/WebClientConfigurator.java ================================================ package com.provectus.kafka.ui.util; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.exception.ValidationException; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import java.io.FileInputStream; import java.security.KeyStore; import java.util.function.Consumer; import javax.annotation.Nullable; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.TrustManagerFactory; import lombok.SneakyThrows; import org.openapitools.jackson.nullable.JsonNullableModule; import org.springframework.http.MediaType; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.http.codec.ClientCodecConfigurer; import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.util.ResourceUtils; import org.springframework.util.unit.DataSize; import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; public class WebClientConfigurator { private final WebClient.Builder builder = WebClient.builder(); private HttpClient httpClient = HttpClient .create() .proxyWithSystemProperties(); public WebClientConfigurator() { configureObjectMapper(defaultOM()); } private static ObjectMapper defaultOM() { return new ObjectMapper() .registerModule(new JavaTimeModule()) .registerModule(new JsonNullableModule()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } public WebClientConfigurator configureSsl(@Nullable ClustersProperties.TruststoreConfig truststoreConfig, @Nullable ClustersProperties.KeystoreConfig keystoreConfig) { return configureSsl( keystoreConfig != null ? keystoreConfig.getKeystoreLocation() : null, keystoreConfig != null ? keystoreConfig.getKeystorePassword() : null, truststoreConfig != null ? truststoreConfig.getTruststoreLocation() : null, truststoreConfig != null ? truststoreConfig.getTruststorePassword() : null ); } @SneakyThrows private WebClientConfigurator configureSsl( @Nullable String keystoreLocation, @Nullable String keystorePassword, @Nullable String truststoreLocation, @Nullable String truststorePassword) { if (truststoreLocation == null && keystoreLocation == null) { return this; } SslContextBuilder contextBuilder = SslContextBuilder.forClient(); if (truststoreLocation != null && truststorePassword != null) { KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load( new FileInputStream((ResourceUtils.getFile(truststoreLocation))), truststorePassword.toCharArray() ); TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); trustManagerFactory.init(trustStore); contextBuilder.trustManager(trustManagerFactory); } // Prepare keystore only if we got a keystore if (keystoreLocation != null && keystorePassword != null) { KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); keyStore.load( new FileInputStream(ResourceUtils.getFile(keystoreLocation)), keystorePassword.toCharArray() ); KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(keyStore, keystorePassword.toCharArray()); contextBuilder.keyManager(keyManagerFactory); } // Create webclient SslContext context = contextBuilder.build(); httpClient = httpClient.secure(t -> t.sslContext(context)); return this; } public WebClientConfigurator configureBasicAuth(@Nullable String username, @Nullable String password) { if (username != null && password != null) { builder.defaultHeaders(httpHeaders -> httpHeaders.setBasicAuth(username, password)); } else if (username != null) { throw new ValidationException("You specified username but did not specify password"); } else if (password != null) { throw new ValidationException("You specified password but did not specify username"); } return this; } public WebClientConfigurator configureBufferSize(DataSize maxBuffSize) { builder.codecs(c -> c.defaultCodecs().maxInMemorySize((int) maxBuffSize.toBytes())); return this; } public WebClientConfigurator configureObjectMapper(ObjectMapper mapper) { builder.codecs(codecs -> { codecs.defaultCodecs() .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper, MediaType.APPLICATION_JSON)); codecs.defaultCodecs() .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, MediaType.APPLICATION_JSON)); }); return this; } public WebClientConfigurator configureCodecs(Consumer configurer) { builder.codecs(configurer); return this; } public WebClient build() { return builder.clientConnector(new ReactorClientHttpConnector(httpClient)).build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/annotation/KafkaClientInternalsDependant.java ================================================ package com.provectus.kafka.ui.util.annotation; /** * All code places that depend on kafka-client's internals or implementation-specific logic * should be marked with this annotation to make further update process easier. */ public @interface KafkaClientInternalsDependant { String value() default ""; } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/AnyFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; // Specifies field that can contain any kind of value - primitive, complex and nulls class AnyFieldSchema implements FieldSchema { static AnyFieldSchema get() { return new AnyFieldSchema(); } private AnyFieldSchema() { } @Override public JsonNode toJsonNode(ObjectMapper mapper) { var arr = mapper.createArrayNode(); arr.add("number"); arr.add("string"); arr.add("object"); arr.add("array"); arr.add("boolean"); arr.add("null"); return mapper.createObjectNode().set("type", arr); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/ArrayFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; class ArrayFieldSchema implements FieldSchema { private final FieldSchema itemsSchema; ArrayFieldSchema(FieldSchema itemsSchema) { this.itemsSchema = itemsSchema; } @Override public JsonNode toJsonNode(ObjectMapper mapper) { final ObjectNode objectNode = mapper.createObjectNode(); objectNode.setAll(new SimpleJsonType(JsonType.Type.ARRAY).toJsonNode(mapper)); objectNode.set("items", itemsSchema.toJsonNode(mapper)); return objectNode; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/AvroJsonSchemaConverter.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import java.net.URI; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import org.apache.avro.Schema; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; public class AvroJsonSchemaConverter implements JsonSchemaConverter { @Override public JsonSchema convert(URI basePath, Schema schema) { final JsonSchema.JsonSchemaBuilder builder = JsonSchema.builder(); builder.id(basePath.resolve(schema.getName())); JsonType type = convertType(schema); builder.type(type); Map definitions = new HashMap<>(); final FieldSchema root = convertSchema(schema, definitions, true); builder.definitions(definitions); if (type.getType().equals(JsonType.Type.OBJECT)) { final ObjectFieldSchema objectRoot = (ObjectFieldSchema) root; builder.properties(objectRoot.getProperties()); builder.required(objectRoot.getRequired()); } return builder.build(); } private FieldSchema convertField(Schema.Field field, Map definitions) { return convertSchema(field.schema(), definitions, false); } private FieldSchema convertSchema(Schema schema, Map definitions, boolean isRoot) { Optional logicalTypeSchema = JsonAvroConversion.LogicalTypeConversion.getJsonSchema(schema); if (logicalTypeSchema.isPresent()) { return logicalTypeSchema.get(); } if (!schema.isUnion()) { JsonType type = convertType(schema); switch (type.getType()) { case BOOLEAN: case NULL: case STRING: case ENUM: case NUMBER: case INTEGER: return new SimpleFieldSchema(type); case OBJECT: if (schema.getType().equals(Schema.Type.MAP)) { return new MapFieldSchema(convertSchema(schema.getValueType(), definitions, isRoot)); } else { return createObjectSchema(schema, definitions, isRoot); } case ARRAY: return createArraySchema(schema, definitions); default: throw new RuntimeException("Unknown type"); } } else { return createUnionSchema(schema, definitions); } } // this method formats json-schema field in a way // to fit avro-> json encoding rules (https://avro.apache.org/docs/1.11.1/specification/_print/#json-encoding) private FieldSchema createUnionSchema(Schema schema, Map definitions) { final boolean nullable = schema.getTypes().stream() .anyMatch(t -> t.getType().equals(Schema.Type.NULL)); final Map fields = schema.getTypes().stream() .filter(t -> !t.getType().equals(Schema.Type.NULL)) .map(f -> { String oneOfFieldName; if (f.getType().equals(Schema.Type.RECORD)) { // for records using full record name oneOfFieldName = f.getFullName(); } else { // for primitive types - using type name oneOfFieldName = f.getType().getName().toLowerCase(); } return Tuples.of(oneOfFieldName, convertSchema(f, definitions, false)); }).collect(Collectors.toMap( Tuple2::getT1, Tuple2::getT2 )); if (nullable) { return new OneOfFieldSchema( List.of( new SimpleFieldSchema(new SimpleJsonType(JsonType.Type.NULL)), new ObjectFieldSchema(fields, Collections.emptyList()) ) ); } else { return new ObjectFieldSchema(fields, Collections.emptyList()); } } private FieldSchema createObjectSchema(Schema schema, Map definitions, boolean isRoot) { var definitionName = schema.getFullName(); if (definitions.containsKey(definitionName)) { return createRefField(definitionName); } // adding stub record, need to avoid infinite recursion definitions.put(definitionName, ObjectFieldSchema.EMPTY); final Map fields = schema.getFields().stream() .map(f -> Tuples.of(f.name(), convertField(f, definitions))) .collect(Collectors.toMap( Tuple2::getT1, Tuple2::getT2 )); final List required = schema.getFields().stream() .filter(f -> !f.schema().isNullable()) .map(Schema.Field::name).collect(Collectors.toList()); var objectSchema = new ObjectFieldSchema(fields, required); if (isRoot) { // replacing stub with self-reference (need for usage in json-schema's oneOf) definitions.put(definitionName, new RefFieldSchema("#")); return objectSchema; } else { // replacing stub record with actual object structure definitions.put(definitionName, objectSchema); return createRefField(definitionName); } } private RefFieldSchema createRefField(String definitionName) { return new RefFieldSchema(String.format("#/definitions/%s", definitionName)); } private ArrayFieldSchema createArraySchema(Schema schema, Map definitions) { return new ArrayFieldSchema( convertSchema(schema.getElementType(), definitions, false) ); } private JsonType convertType(Schema schema) { return switch (schema.getType()) { case INT, LONG -> new SimpleJsonType(JsonType.Type.INTEGER); case MAP, RECORD -> new SimpleJsonType(JsonType.Type.OBJECT); case ENUM -> new EnumJsonType(schema.getEnumSymbols()); case BYTES, STRING -> new SimpleJsonType(JsonType.Type.STRING); case NULL -> new SimpleJsonType(JsonType.Type.NULL); case ARRAY -> new SimpleJsonType(JsonType.Type.ARRAY); case FIXED, FLOAT, DOUBLE -> new SimpleJsonType(JsonType.Type.NUMBER); case BOOLEAN -> new SimpleJsonType(JsonType.Type.BOOLEAN); default -> new SimpleJsonType(JsonType.Type.STRING); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/EnumJsonType.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.TextNode; import java.util.List; import java.util.Map; class EnumJsonType extends JsonType { private final List values; EnumJsonType(List values) { super(Type.ENUM); this.values = values; } @Override public Map toJsonNode(ObjectMapper mapper) { return Map.of( "type", new TextNode(Type.STRING.getName()), Type.ENUM.getName(), mapper.valueToTree(values) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/FieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; interface FieldSchema { JsonNode toJsonNode(ObjectMapper mapper); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/JsonAvroConversion.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.DoubleNode; import com.fasterxml.jackson.databind.node.FloatNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.Lists; import com.provectus.kafka.ui.exception.JsonAvroConversionException; import io.confluent.kafka.serializers.AvroData; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.stream.Stream; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; // json <-> avro public class JsonAvroConversion { private static final JsonMapper MAPPER = new JsonMapper(); private static final Schema NULL_SCHEMA = Schema.create(Schema.Type.NULL); private static final String FORMAT = "format"; private static final String DATE_TIME = "date-time"; // converts json into Object that is expected input for KafkaAvroSerializer // (with AVRO_USE_LOGICAL_TYPE_CONVERTERS flat enabled!) public static Object convertJsonToAvro(String jsonString, Schema avroSchema) { JsonNode rootNode = null; try { rootNode = MAPPER.readTree(jsonString); } catch (JsonProcessingException e) { throw new JsonAvroConversionException("String is not a valid json"); } return convert(rootNode, avroSchema); } private static Object convert(JsonNode node, Schema avroSchema) { return switch (avroSchema.getType()) { case RECORD -> { assertJsonType(node, JsonNodeType.OBJECT); var rec = new GenericData.Record(avroSchema); for (Schema.Field field : avroSchema.getFields()) { if (node.has(field.name()) && !node.get(field.name()).isNull()) { rec.put(field.name(), convert(node.get(field.name()), field.schema())); } } yield rec; } case MAP -> { assertJsonType(node, JsonNodeType.OBJECT); var map = new LinkedHashMap(); var valueSchema = avroSchema.getValueType(); node.fields().forEachRemaining(f -> map.put(f.getKey(), convert(f.getValue(), valueSchema))); yield map; } case ARRAY -> { assertJsonType(node, JsonNodeType.ARRAY); var lst = new ArrayList<>(); node.elements().forEachRemaining(e -> lst.add(convert(e, avroSchema.getElementType()))); yield lst; } case ENUM -> { assertJsonType(node, JsonNodeType.STRING); String symbol = node.textValue(); if (!avroSchema.getEnumSymbols().contains(symbol)) { throw new JsonAvroConversionException("%s is not a part of enum symbols [%s]" .formatted(symbol, avroSchema.getEnumSymbols())); } yield new GenericData.EnumSymbol(avroSchema, symbol); } case UNION -> { // for types from enum (other than null) payload should be an object with single key == name of type // ex: schema = [ "null", "int", "string" ], possible payloads = null, { "string": "str" }, { "int": 123 } if (node.isNull() && avroSchema.getTypes().contains(NULL_SCHEMA)) { yield null; } assertJsonType(node, JsonNodeType.OBJECT); var elements = Lists.newArrayList(node.fields()); if (elements.size() != 1) { throw new JsonAvroConversionException( "UNION field value should be an object with single field == type name"); } Map.Entry typeNameToValue = elements.get(0); List candidates = new ArrayList<>(); for (Schema unionType : avroSchema.getTypes()) { if (typeNameToValue.getKey().equals(unionType.getFullName())) { yield convert(typeNameToValue.getValue(), unionType); } if (typeNameToValue.getKey().equals(unionType.getName())) { candidates.add(unionType); } } if (candidates.size() == 1) { yield convert(typeNameToValue.getValue(), candidates.get(0)); } if (candidates.size() > 1) { throw new JsonAvroConversionException( "Can't select type within union for value '%s'. Provide full type name.".formatted(node) ); } throw new JsonAvroConversionException( "json value '%s' is cannot be converted to any of union types [%s]" .formatted(node, avroSchema.getTypes())); } case STRING -> { if (isLogicalType(avroSchema)) { yield processLogicalType(node, avroSchema); } assertJsonType(node, JsonNodeType.STRING); yield node.textValue(); } case LONG -> { if (isLogicalType(avroSchema)) { yield processLogicalType(node, avroSchema); } assertJsonType(node, JsonNodeType.NUMBER); assertJsonNumberType(node, JsonParser.NumberType.LONG, JsonParser.NumberType.INT); yield node.longValue(); } case INT -> { if (isLogicalType(avroSchema)) { yield processLogicalType(node, avroSchema); } assertJsonType(node, JsonNodeType.NUMBER); assertJsonNumberType(node, JsonParser.NumberType.INT); yield node.intValue(); } case FLOAT -> { assertJsonType(node, JsonNodeType.NUMBER); assertJsonNumberType(node, JsonParser.NumberType.DOUBLE, JsonParser.NumberType.FLOAT); yield node.floatValue(); } case DOUBLE -> { assertJsonType(node, JsonNodeType.NUMBER); assertJsonNumberType(node, JsonParser.NumberType.DOUBLE, JsonParser.NumberType.FLOAT); yield node.doubleValue(); } case BOOLEAN -> { assertJsonType(node, JsonNodeType.BOOLEAN); yield node.booleanValue(); } case NULL -> { assertJsonType(node, JsonNodeType.NULL); yield null; } case BYTES -> { if (isLogicalType(avroSchema)) { yield processLogicalType(node, avroSchema); } assertJsonType(node, JsonNodeType.STRING); // logic copied from JsonDecoder::readBytes yield ByteBuffer.wrap(node.textValue().getBytes(StandardCharsets.ISO_8859_1)); } case FIXED -> { if (isLogicalType(avroSchema)) { yield processLogicalType(node, avroSchema); } assertJsonType(node, JsonNodeType.STRING); byte[] bytes = node.textValue().getBytes(StandardCharsets.ISO_8859_1); if (bytes.length != avroSchema.getFixedSize()) { throw new JsonAvroConversionException( "Fixed field has unexpected size %d (should be %d)" .formatted(bytes.length, avroSchema.getFixedSize())); } yield new GenericData.Fixed(avroSchema, bytes); } }; } // converts output of KafkaAvroDeserializer (with AVRO_USE_LOGICAL_TYPE_CONVERTERS flat enabled!) into json. // Note: conversion should be compatible with AvroJsonSchemaConverter logic! public static JsonNode convertAvroToJson(Object obj, Schema avroSchema) { if (obj == null) { return NullNode.getInstance(); } return switch (avroSchema.getType()) { case RECORD -> { var rec = (GenericData.Record) obj; ObjectNode node = MAPPER.createObjectNode(); for (Schema.Field field : avroSchema.getFields()) { var fieldVal = rec.get(field.name()); if (fieldVal != null) { node.set(field.name(), convertAvroToJson(fieldVal, field.schema())); } } yield node; } case MAP -> { ObjectNode node = MAPPER.createObjectNode(); ((Map) obj).forEach((k, v) -> node.set(k.toString(), convertAvroToJson(v, avroSchema.getValueType()))); yield node; } case ARRAY -> { var list = (List) obj; ArrayNode node = MAPPER.createArrayNode(); list.forEach(e -> node.add(convertAvroToJson(e, avroSchema.getElementType()))); yield node; } case ENUM -> { yield new TextNode(obj.toString()); } case UNION -> { ObjectNode node = MAPPER.createObjectNode(); int unionIdx = AvroData.getGenericData().resolveUnion(avroSchema, obj); Schema selectedType = avroSchema.getTypes().get(unionIdx); node.set( selectUnionTypeFieldName(avroSchema, selectedType, unionIdx), convertAvroToJson(obj, selectedType) ); yield node; } case STRING -> { if (isLogicalType(avroSchema)) { yield processLogicalType(obj, avroSchema); } yield new TextNode(obj.toString()); } case LONG -> { if (isLogicalType(avroSchema)) { yield processLogicalType(obj, avroSchema); } yield new LongNode((Long) obj); } case INT -> { if (isLogicalType(avroSchema)) { yield processLogicalType(obj, avroSchema); } yield new IntNode((Integer) obj); } case FLOAT -> new FloatNode((Float) obj); case DOUBLE -> new DoubleNode((Double) obj); case BOOLEAN -> BooleanNode.valueOf((Boolean) obj); case NULL -> NullNode.getInstance(); case BYTES -> { if (isLogicalType(avroSchema)) { yield processLogicalType(obj, avroSchema); } ByteBuffer bytes = (ByteBuffer) obj; //see JsonEncoder::writeByteArray yield new TextNode(new String(bytes.array(), StandardCharsets.ISO_8859_1)); } case FIXED -> { if (isLogicalType(avroSchema)) { yield processLogicalType(obj, avroSchema); } var fixed = (GenericData.Fixed) obj; yield new TextNode(new String(fixed.bytes(), StandardCharsets.ISO_8859_1)); } }; } // select name for a key field that represents type name of union. // For records selects short name, if it is possible. private static String selectUnionTypeFieldName(Schema unionSchema, Schema chosenType, int chosenTypeIdx) { var types = unionSchema.getTypes(); if (types.size() == 2 && types.contains(NULL_SCHEMA)) { return chosenType.getName(); } for (int i = 0; i < types.size(); i++) { if (i != chosenTypeIdx && chosenType.getName().equals(types.get(i).getName())) { // there is another type inside union with the same name // so, we have to use fullname return chosenType.getFullName(); } } return chosenType.getName(); } private static Object processLogicalType(JsonNode node, Schema schema) { return findConversion(schema) .map(c -> c.jsonToAvroConversion.apply(node, schema)) .orElseThrow(() -> new JsonAvroConversionException("'%s' logical type is not supported" .formatted(schema.getLogicalType().getName()))); } private static JsonNode processLogicalType(Object obj, Schema schema) { return findConversion(schema) .map(c -> c.avroToJsonConversion.apply(obj, schema)) .orElseThrow(() -> new JsonAvroConversionException("'%s' logical type is not supported" .formatted(schema.getLogicalType().getName()))); } private static Optional findConversion(Schema schema) { String logicalTypeName = schema.getLogicalType().getName(); return Stream.of(LogicalTypeConversion.values()) .filter(t -> t.name.equalsIgnoreCase(logicalTypeName)) .findFirst(); } private static boolean isLogicalType(Schema schema) { return schema.getLogicalType() != null; } private static void assertJsonType(JsonNode node, JsonNodeType... allowedTypes) { if (Stream.of(allowedTypes).noneMatch(t -> node.getNodeType() == t)) { throw new JsonAvroConversionException( "%s node has unexpected type, allowed types %s, actual type %s" .formatted(node, Arrays.toString(allowedTypes), node.getNodeType())); } } private static void assertJsonNumberType(JsonNode node, JsonParser.NumberType... allowedTypes) { if (Stream.of(allowedTypes).noneMatch(t -> node.numberType() == t)) { throw new JsonAvroConversionException( "%s node has unexpected numeric type, allowed types %s, actual type %s" .formatted(node, Arrays.toString(allowedTypes), node.numberType())); } } enum LogicalTypeConversion { UUID("uuid", (node, schema) -> { assertJsonType(node, JsonNodeType.STRING); return java.util.UUID.fromString(node.asText()); }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode("uuid")))) ), DECIMAL("decimal", (node, schema) -> { if (node.isTextual()) { return new BigDecimal(node.asText()); } else if (node.isNumber()) { return new BigDecimal(node.numberValue().toString()); } throw new JsonAvroConversionException( "node '%s' can't be converted to decimal logical type" .formatted(node)); }, (obj, schema) -> { return new DecimalNode((BigDecimal) obj); }, new SimpleFieldSchema(new SimpleJsonType(JsonType.Type.NUMBER)) ), DATE("date", (node, schema) -> { if (node.isInt()) { return LocalDate.ofEpochDay(node.intValue()); } else if (node.isTextual()) { return LocalDate.parse(node.asText()); } else { throw new JsonAvroConversionException( "node '%s' can't be converted to date logical type" .formatted(node)); } }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode("date")))) ), TIME_MILLIS("time-millis", (node, schema) -> { if (node.isIntegralNumber()) { return LocalTime.ofNanoOfDay(TimeUnit.MILLISECONDS.toNanos(node.longValue())); } else if (node.isTextual()) { return LocalTime.parse(node.asText()); } else { throw new JsonAvroConversionException( "node '%s' can't be converted to time-millis logical type" .formatted(node)); } }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode("time")))) ), TIME_MICROS("time-micros", (node, schema) -> { if (node.isIntegralNumber()) { return LocalTime.ofNanoOfDay(TimeUnit.MICROSECONDS.toNanos(node.longValue())); } else if (node.isTextual()) { return LocalTime.parse(node.asText()); } else { throw new JsonAvroConversionException( "node '%s' can't be converted to time-micros logical type" .formatted(node)); } }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode("time")))) ), TIMESTAMP_MILLIS("timestamp-millis", (node, schema) -> { if (node.isIntegralNumber()) { return Instant.ofEpochMilli(node.longValue()); } else if (node.isTextual()) { return Instant.parse(node.asText()); } else { throw new JsonAvroConversionException( "node '%s' can't be converted to timestamp-millis logical type" .formatted(node)); } }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode(DATE_TIME)))) ), TIMESTAMP_MICROS("timestamp-micros", (node, schema) -> { if (node.isIntegralNumber()) { // TimeConversions.TimestampMicrosConversion for impl long microsFromEpoch = node.longValue(); long epochSeconds = microsFromEpoch / (1_000_000L); long nanoAdjustment = (microsFromEpoch % (1_000_000L)) * 1_000L; return Instant.ofEpochSecond(epochSeconds, nanoAdjustment); } else if (node.isTextual()) { return Instant.parse(node.asText()); } else { throw new JsonAvroConversionException( "node '%s' can't be converted to timestamp-millis logical type" .formatted(node)); } }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode(DATE_TIME)))) ), LOCAL_TIMESTAMP_MILLIS("local-timestamp-millis", (node, schema) -> { if (node.isTextual()) { return LocalDateTime.parse(node.asText()); } // TimeConversions.TimestampMicrosConversion for impl Instant instant = (Instant) TIMESTAMP_MILLIS.jsonToAvroConversion.apply(node, schema); return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode(DATE_TIME)))) ), LOCAL_TIMESTAMP_MICROS("local-timestamp-micros", (node, schema) -> { if (node.isTextual()) { return LocalDateTime.parse(node.asText()); } Instant instant = (Instant) TIMESTAMP_MICROS.jsonToAvroConversion.apply(node, schema); return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); }, (obj, schema) -> { return new TextNode(obj.toString()); }, new SimpleFieldSchema( new SimpleJsonType( JsonType.Type.STRING, Map.of(FORMAT, new TextNode(DATE_TIME)))) ); private final String name; private final BiFunction jsonToAvroConversion; private final BiFunction avroToJsonConversion; private final FieldSchema jsonSchema; LogicalTypeConversion(String name, BiFunction jsonToAvroConversion, BiFunction avroToJsonConversion, FieldSchema jsonSchema) { this.name = name; this.jsonToAvroConversion = jsonToAvroConversion; this.avroToJsonConversion = avroToJsonConversion; this.jsonSchema = jsonSchema; } static Optional getJsonSchema(Schema schema) { if (schema.getLogicalType() == null) { return Optional.empty(); } String logicalTypeName = schema.getLogicalType().getName(); return Stream.of(JsonAvroConversion.LogicalTypeConversion.values()) .filter(t -> t.name.equalsIgnoreCase(logicalTypeName)) .map(c -> c.jsonSchema) .findFirst(); } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/JsonSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import java.net.URI; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.Builder; import lombok.Data; import lombok.SneakyThrows; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; @Data @Builder public class JsonSchema { private final URI id; private final URI schema = URI.create("https://json-schema.org/draft/2020-12/schema"); private final String title; private final JsonType type; private final Map properties; private final Map definitions; private final List required; private final String rootRef; public String toJson() { final ObjectMapper mapper = new ObjectMapper(); final ObjectNode objectNode = mapper.createObjectNode(); objectNode.set("$id", new TextNode(id.toString())); objectNode.set("$schema", new TextNode(schema.toString())); objectNode.setAll(type.toJsonNode(mapper)); if (properties != null && !properties.isEmpty()) { objectNode.set("properties", mapper.valueToTree( properties.entrySet().stream() .map(e -> Tuples.of(e.getKey(), e.getValue().toJsonNode(mapper))) .collect(Collectors.toMap( Tuple2::getT1, Tuple2::getT2 )) )); if (!required.isEmpty()) { objectNode.set("required", mapper.valueToTree(required)); } } if (definitions != null && !definitions.isEmpty()) { objectNode.set("definitions", mapper.valueToTree( definitions.entrySet().stream() .map(e -> Tuples.of(e.getKey(), e.getValue().toJsonNode(mapper))) .collect(Collectors.toMap( Tuple2::getT1, Tuple2::getT2 )) )); } if (rootRef != null) { objectNode.set("$ref", new TextNode(rootRef)); } return objectNode.toString(); } @SneakyThrows public static JsonSchema stringSchema() { return JsonSchema.builder() .id(new URI("http://unknown.unknown")) .type(new SimpleJsonType(JsonType.Type.STRING)) .build(); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/JsonSchemaConverter.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import java.net.URI; public interface JsonSchemaConverter { JsonSchema convert(URI basePath, T schema); } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/JsonType.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Map; abstract class JsonType { protected final Type type; protected JsonType(Type type) { this.type = type; } Type getType() { return type; } abstract Map toJsonNode(ObjectMapper mapper); enum Type { NULL, BOOLEAN, OBJECT, ARRAY, NUMBER, INTEGER, ENUM, STRING; private final String name; Type() { this.name = this.name().toLowerCase(); } public String getName() { return name; } } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/MapFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import javax.annotation.Nullable; class MapFieldSchema implements FieldSchema { private final @Nullable FieldSchema itemSchema; MapFieldSchema(@Nullable FieldSchema itemSchema) { this.itemSchema = itemSchema; } MapFieldSchema() { this(null); } @Override public JsonNode toJsonNode(ObjectMapper mapper) { final ObjectNode objectNode = mapper.createObjectNode(); objectNode.set("type", new TextNode(JsonType.Type.OBJECT.getName())); objectNode.set("additionalProperties", itemSchema != null ? itemSchema.toJsonNode(mapper) : BooleanNode.TRUE); return objectNode; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/ObjectFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; class ObjectFieldSchema implements FieldSchema { static final ObjectFieldSchema EMPTY = new ObjectFieldSchema(Map.of(), List.of()); private final Map properties; private final List required; ObjectFieldSchema(Map properties, List required) { this.properties = properties; this.required = required; } Map getProperties() { return properties; } List getRequired() { return required; } @Override public JsonNode toJsonNode(ObjectMapper mapper) { final Map nodes = properties.entrySet().stream() .map(e -> Tuples.of(e.getKey(), e.getValue().toJsonNode(mapper))) .collect(Collectors.toMap( Tuple2::getT1, Tuple2::getT2 )); final ObjectNode objectNode = mapper.createObjectNode(); objectNode.setAll( new SimpleJsonType(JsonType.Type.OBJECT).toJsonNode(mapper) ); objectNode.set("properties", mapper.valueToTree(nodes)); if (!required.isEmpty()) { objectNode.set("required", mapper.valueToTree(required)); } return objectNode; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/OneOfFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import java.util.stream.Collectors; class OneOfFieldSchema implements FieldSchema { private final List schemaList; OneOfFieldSchema(List schemaList) { this.schemaList = schemaList; } @Override public JsonNode toJsonNode(ObjectMapper mapper) { return mapper.createObjectNode() .set("oneOf", mapper.createArrayNode().addAll( schemaList.stream() .map(s -> s.toJsonNode(mapper)) .collect(Collectors.toList()) ) ); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/ProtobufSchemaConverter.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import static java.util.Objects.requireNonNull; import com.fasterxml.jackson.databind.node.BigIntegerNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.primitives.UnsignedInteger; import com.google.common.primitives.UnsignedLong; import com.google.protobuf.Any; import com.google.protobuf.BoolValue; import com.google.protobuf.BytesValue; import com.google.protobuf.Descriptors; import com.google.protobuf.DoubleValue; import com.google.protobuf.Duration; import com.google.protobuf.FieldMask; import com.google.protobuf.FloatValue; import com.google.protobuf.Int32Value; import com.google.protobuf.Int64Value; import com.google.protobuf.ListValue; import com.google.protobuf.StringValue; import com.google.protobuf.Struct; import com.google.protobuf.Timestamp; import com.google.protobuf.UInt32Value; import com.google.protobuf.UInt64Value; import com.google.protobuf.Value; import java.net.URI; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; public class ProtobufSchemaConverter implements JsonSchemaConverter { private static final String MAXIMUM = "maximum"; private static final String MINIMUM = "minimum"; private final Set simpleTypesWrapperNames = Set.of( BoolValue.getDescriptor().getFullName(), Int32Value.getDescriptor().getFullName(), UInt32Value.getDescriptor().getFullName(), Int64Value.getDescriptor().getFullName(), UInt64Value.getDescriptor().getFullName(), StringValue.getDescriptor().getFullName(), BytesValue.getDescriptor().getFullName(), FloatValue.getDescriptor().getFullName(), DoubleValue.getDescriptor().getFullName() ); @Override public JsonSchema convert(URI basePath, Descriptors.Descriptor schema) { Map definitions = new HashMap<>(); RefFieldSchema rootRef = registerObjectAndReturnRef(schema, definitions); return JsonSchema.builder() .id(basePath.resolve(schema.getFullName())) .type(new SimpleJsonType(JsonType.Type.OBJECT)) .rootRef(rootRef.getRef()) .definitions(definitions) .build(); } private RefFieldSchema registerObjectAndReturnRef(Descriptors.Descriptor schema, Map definitions) { var definition = schema.getFullName(); if (definitions.containsKey(definition)) { return createRefField(definition); } // adding stub record, need to avoid infinite recursion definitions.put(definition, ObjectFieldSchema.EMPTY); Map fields = schema.getFields().stream() .map(f -> Tuples.of(f.getName(), convertField(f, definitions))) .collect(Collectors.toMap(Tuple2::getT1, Tuple2::getT2)); List required = schema.getFields().stream() .filter(Descriptors.FieldDescriptor::isRequired) .map(Descriptors.FieldDescriptor::getName) .collect(Collectors.toList()); // replacing stub record with actual object structure definitions.put(definition, new ObjectFieldSchema(fields, required)); return createRefField(definition); } private RefFieldSchema createRefField(String definition) { return new RefFieldSchema("#/definitions/%s".formatted(definition)); } private FieldSchema convertField(Descriptors.FieldDescriptor field, Map definitions) { Optional wellKnownTypeSchema = convertProtoWellKnownTypes(field); if (wellKnownTypeSchema.isPresent()) { return wellKnownTypeSchema.get(); } if (field.isMapField()) { return new MapFieldSchema(); } final JsonType jsonType = convertType(field); FieldSchema fieldSchema; if (jsonType.getType().equals(JsonType.Type.OBJECT)) { fieldSchema = registerObjectAndReturnRef(field.getMessageType(), definitions); } else { fieldSchema = new SimpleFieldSchema(jsonType); } if (field.isRepeated()) { return new ArrayFieldSchema(fieldSchema); } else { return fieldSchema; } } // converts Protobuf Well-known type (from google.protobuf.* packages) to Json-schema types // see JsonFormat::buildWellKnownTypePrinters for impl details private Optional convertProtoWellKnownTypes(Descriptors.FieldDescriptor field) { // all well-known types are messages if (field.getType() != Descriptors.FieldDescriptor.Type.MESSAGE) { return Optional.empty(); } String typeName = field.getMessageType().getFullName(); if (typeName.equals(Timestamp.getDescriptor().getFullName())) { return Optional.of( new SimpleFieldSchema( new SimpleJsonType(JsonType.Type.STRING, Map.of("format", new TextNode("date-time"))))); } if (typeName.equals(Duration.getDescriptor().getFullName())) { return Optional.of( new SimpleFieldSchema( //TODO: current UI is failing when format=duration is set - need to fix this first new SimpleJsonType(JsonType.Type.STRING // , Map.of("format", new TextNode("duration")) ))); } if (typeName.equals(FieldMask.getDescriptor().getFullName())) { return Optional.of(new SimpleFieldSchema(new SimpleJsonType(JsonType.Type.STRING))); } if (typeName.equals(Any.getDescriptor().getFullName()) || typeName.equals(Struct.getDescriptor().getFullName())) { return Optional.of(ObjectFieldSchema.EMPTY); } if (typeName.equals(Value.getDescriptor().getFullName())) { return Optional.of(AnyFieldSchema.get()); } if (typeName.equals(ListValue.getDescriptor().getFullName())) { return Optional.of(new ArrayFieldSchema(AnyFieldSchema.get())); } if (simpleTypesWrapperNames.contains(typeName)) { return Optional.of(new SimpleFieldSchema( convertType(requireNonNull(field.getMessageType().findFieldByName("value"))))); } return Optional.empty(); } private JsonType convertType(Descriptors.FieldDescriptor field) { return switch (field.getType()) { case INT32, FIXED32, SFIXED32, SINT32 -> new SimpleJsonType( JsonType.Type.INTEGER, Map.of( MAXIMUM, IntNode.valueOf(Integer.MAX_VALUE), MINIMUM, IntNode.valueOf(Integer.MIN_VALUE) ) ); case UINT32 -> new SimpleJsonType( JsonType.Type.INTEGER, Map.of( MAXIMUM, LongNode.valueOf(UnsignedInteger.MAX_VALUE.longValue()), MINIMUM, IntNode.valueOf(0) ) ); //TODO: actually all *64 types will be printed with quotes (as strings), // see JsonFormat::printSingleFieldValue for impl. This can cause problems when you copy-paste from messages // table to `Produce` area - need to think if it is critical or not. case INT64, FIXED64, SFIXED64, SINT64 -> new SimpleJsonType( JsonType.Type.INTEGER, Map.of( MAXIMUM, LongNode.valueOf(Long.MAX_VALUE), MINIMUM, LongNode.valueOf(Long.MIN_VALUE) ) ); case UINT64 -> new SimpleJsonType( JsonType.Type.INTEGER, Map.of( MAXIMUM, new BigIntegerNode(UnsignedLong.MAX_VALUE.bigIntegerValue()), MINIMUM, LongNode.valueOf(0) ) ); case MESSAGE, GROUP -> new SimpleJsonType(JsonType.Type.OBJECT); case ENUM -> new EnumJsonType( field.getEnumType().getValues().stream() .map(Descriptors.EnumValueDescriptor::getName) .collect(Collectors.toList()) ); case BYTES, STRING -> new SimpleJsonType(JsonType.Type.STRING); case FLOAT, DOUBLE -> new SimpleJsonType(JsonType.Type.NUMBER); case BOOL -> new SimpleJsonType(JsonType.Type.BOOLEAN); }; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/RefFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.TextNode; class RefFieldSchema implements FieldSchema { private final String ref; RefFieldSchema(String ref) { this.ref = ref; } @Override public JsonNode toJsonNode(ObjectMapper mapper) { return mapper.createObjectNode().set("$ref", new TextNode(ref)); } String getRef() { return ref; } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/SimpleFieldSchema.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; class SimpleFieldSchema implements FieldSchema { private final JsonType type; SimpleFieldSchema(JsonType type) { this.type = type; } @Override public JsonNode toJsonNode(ObjectMapper mapper) { return mapper.createObjectNode().setAll(type.toJsonNode(mapper)); } } ================================================ FILE: kafka-ui-api/src/main/java/com/provectus/kafka/ui/util/jsonschema/SimpleJsonType.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.ImmutableMap; import java.util.Map; class SimpleJsonType extends JsonType { private final Map additionalTypeProperties; SimpleJsonType(Type type) { this(type, Map.of()); } SimpleJsonType(Type type, Map additionalTypeProperties) { super(type); this.additionalTypeProperties = additionalTypeProperties; } @Override public Map toJsonNode(ObjectMapper mapper) { return ImmutableMap.builder() .put("type", new TextNode(type.getName())) .putAll(additionalTypeProperties) .build(); } } ================================================ FILE: kafka-ui-api/src/main/resources/application-local.yml ================================================ logging: level: root: INFO com.provectus: DEBUG #org.springframework.http.codec.json.Jackson2JsonEncoder: DEBUG #org.springframework.http.codec.json.Jackson2JsonDecoder: DEBUG reactor.netty.http.server.AccessLog: INFO org.springframework.security: DEBUG #server: # port: 8080 #- Port in which kafka-ui will run. spring: jmx: enabled: true ldap: urls: ldap://localhost:10389 base: "cn={0},ou=people,dc=planetexpress,dc=com" admin-user: "cn=admin,dc=planetexpress,dc=com" admin-password: "GoodNewsEveryone" user-filter-search-base: "dc=planetexpress,dc=com" user-filter-search-filter: "(&(uid={0})(objectClass=inetOrgPerson))" group-filter-search-base: "ou=people,dc=planetexpress,dc=com" kafka: clusters: - name: local bootstrapServers: localhost:9092 schemaRegistry: http://localhost:8085 ksqldbServer: http://localhost:8088 kafkaConnect: - name: first address: http://localhost:8083 metrics: port: 9997 type: JMX dynamic.config.enabled: true oauth2: ldap: activeDirectory: false aсtiveDirectory.domain: domain.com auth: type: DISABLED # type: OAUTH2 # type: LDAP oauth2: client: cognito: clientId: # CLIENT ID clientSecret: # CLIENT SECRET scope: openid client-name: cognito provider: cognito redirect-uri: http://localhost:8080/login/oauth2/code/cognito authorization-grant-type: authorization_code issuer-uri: https://cognito-idp.eu-central-1.amazonaws.com/eu-central-1_M7cIUn1nj jwk-set-uri: https://cognito-idp.eu-central-1.amazonaws.com/eu-central-1_M7cIUn1nj/.well-known/jwks.json user-name-attribute: cognito:username custom-params: type: cognito logoutUrl: https://kafka-ui.auth.eu-central-1.amazoncognito.com/logout google: provider: google clientId: # CLIENT ID clientSecret: # CLIENT SECRET user-name-attribute: email custom-params: type: google allowedDomain: provectus.com github: provider: github clientId: # CLIENT ID clientSecret: # CLIENT SECRET scope: - read:org user-name-attribute: login custom-params: type: github rbac: roles: - name: "memelords" clusters: - local subjects: - provider: oauth_google type: domain value: "provectus.com" - provider: oauth_google type: user value: "name@provectus.com" - provider: oauth_github type: organization value: "provectus" - provider: oauth_github type: user value: "memelord" - provider: oauth_cognito type: user value: "username" - provider: oauth_cognito type: group value: "memelords" - provider: ldap type: group value: "admin_staff" # NOT IMPLEMENTED YET # - provider: ldap_ad # type: group # value: "admin_staff" permissions: - resource: applicationconfig actions: all - resource: clusterconfig actions: all - resource: topic value: ".*" actions: all - resource: consumer value: ".*" actions: all - resource: schema value: ".*" actions: all - resource: connect value: "*" actions: all - resource: ksql actions: all - resource: acl actions: all - resource: audit actions: all ================================================ FILE: kafka-ui-api/src/main/resources/application.yml ================================================ auth: type: DISABLED management: endpoint: info: enabled: true health: enabled: true endpoints: web: exposure: include: "info,health,prometheus" logging: level: root: INFO com.provectus: DEBUG reactor.netty.http.server.AccessLog: INFO org.hibernate.validator: WARN ================================================ FILE: kafka-ui-api/src/main/resources/banner.txt ================================================ _ _ ___ __ _ _ _ __ __ _ | | | |_ _| / _|___ _ _ /_\ _ __ __ _ __| |_ ___ | |/ /__ _ / _| |_____ | |_| || | | _/ _ | '_| / _ \| '_ / _` / _| ' \/ -_) | ' %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%c{1}): %msg%n%throwable ================================================ FILE: kafka-ui-api/src/main/resources/static/static/css/bootstrap.min.css ================================================ /*! * Bootstrap v4.0.0-beta (https://getbootstrap.com) * Copyright 2011-2017 The Bootstrap Authors * Copyright 2011-2017 Twitter, Inc. * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) */@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a,a:visited{text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #ddd!important}}html{box-sizing:border-box;font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}*,::after,::before{box-sizing:inherit}@-ms-viewport{width:device-width}article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff}[tabindex="-1"]:focus{outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#0056b3;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus,a:not([href]):not([tabindex]):hover{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}code,kbd,pre,samp{font-family:monospace,monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg:not(:root){overflow:hidden}[role=button],a,area,button,input,label,select,summary,textarea{-ms-touch-action:manipulation;touch-action:manipulation}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#868e96;text-align:left;caption-side:bottom}th{text-align:left}label{display:inline-block;margin-bottom:.5rem}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}input[type=date],input[type=datetime-local],input[type=month],input[type=time]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-family:inherit;font-weight:500;line-height:1.1;color:inherit}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.1}.display-2{font-size:5.5rem;font-weight:300;line-height:1.1}.display-3{font-size:4.5rem;font-weight:300;line-height:1.1}.display-4{font-size:3.5rem;font-weight:300;line-height:1.1}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:5px}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#868e96}.blockquote-footer::before{content:"\2014 \00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #ddd;border-radius:.25rem;transition:all .2s ease-in-out;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#868e96}code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}code{padding:.2rem .4rem;font-size:90%;color:#bd4147;background-color:#f8f9fa;border-radius:.25rem}a>code{padding:0;color:inherit;background-color:inherit}kbd{padding:.2rem .4rem;font-size:90%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;margin-top:0;margin-bottom:1rem;font-size:90%;color:#212529}pre code{padding:0;font-size:inherit;color:inherit;background-color:transparent;border-radius:0}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{margin-right:auto;margin-left:auto;padding-right:15px;padding-left:15px;width:100%}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid{width:100%;margin-right:auto;margin-left:auto;padding-right:15px;padding-left:15px;width:100%}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;min-height:1px;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}}.table{width:100%;max-width:100%;margin-bottom:1rem;background-color:transparent}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #e9ecef}.table thead th{vertical-align:bottom;border-bottom:2px solid #e9ecef}.table tbody+tbody{border-top:2px solid #e9ecef}.table .table{background-color:#fff}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #e9ecef}.table-bordered td,.table-bordered th{border:1px solid #e9ecef}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#dddfe2}.table-hover .table-secondary:hover{background-color:#cfd2d6}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#cfd2d6}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.thead-inverse th{color:#fff;background-color:#212529}.thead-default th{color:#495057;background-color:#e9ecef}.table-inverse{color:#fff;background-color:#212529}.table-inverse td,.table-inverse th,.table-inverse thead th{border-color:#32383e}.table-inverse.table-bordered{border:0}.table-inverse.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-inverse.table-hover tbody tr:hover{background-color:rgba(255,255,255,.075)}@media (max-width:991px){.table-responsive{display:block;width:100%;overflow-x:auto;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive.table-bordered{border:0}}.form-control{display:block;width:100%;padding:.5rem .75rem;font-size:1rem;line-height:1.25;color:#495057;background-color:#fff;background-image:none;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;transition:border-color ease-in-out .15s,box-shadow ease-in-out .15s}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0}.form-control::-webkit-input-placeholder{color:#868e96;opacity:1}.form-control:-ms-input-placeholder{color:#868e96;opacity:1}.form-control::placeholder{color:#868e96;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}select.form-control:not([size]):not([multiple]){height:calc(2.25rem + 2px)}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block}.col-form-label{padding-top:calc(.5rem - 1px * 2);padding-bottom:calc(.5rem - 1px * 2);margin-bottom:0}.col-form-label-lg{padding-top:calc(.5rem - 1px * 2);padding-bottom:calc(.5rem - 1px * 2);font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem - 1px * 2);padding-bottom:calc(.25rem - 1px * 2);font-size:.875rem}.col-form-legend{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;font-size:1rem}.form-control-plaintext{padding-top:.5rem;padding-bottom:.5rem;margin-bottom:0;line-height:1.25;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm,.input-group-lg>.form-control-plaintext.form-control,.input-group-lg>.form-control-plaintext.input-group-addon,.input-group-lg>.input-group-btn>.form-control-plaintext.btn,.input-group-sm>.form-control-plaintext.form-control,.input-group-sm>.form-control-plaintext.input-group-addon,.input-group-sm>.input-group-btn>.form-control-plaintext.btn{padding-right:0;padding-left:0}.form-control-sm,.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-sm>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-sm>select.form-control:not([size]):not([multiple]),.input-group-sm>select.input-group-addon:not([size]):not([multiple]),select.form-control-sm:not([size]):not([multiple]){height:calc(1.8125rem + 2px)}.form-control-lg,.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-lg>.input-group-btn>select.btn:not([size]):not([multiple]),.input-group-lg>select.form-control:not([size]):not([multiple]),.input-group-lg>select.input-group-addon:not([size]):not([multiple]),select.form-control-lg:not([size]):not([multiple]){height:calc(2.3125rem + 2px)}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;margin-bottom:.5rem}.form-check.disabled .form-check-label{color:#868e96}.form-check-label{padding-left:1.25rem;margin-bottom:0}.form-check-input{position:absolute;margin-top:.25rem;margin-left:-1.25rem}.form-check-input:only-child{position:static}.form-check-inline{display:inline-block}.form-check-inline .form-check-label{vertical-align:middle}.form-check-inline+.form-check-inline{margin-left:.75rem}.invalid-feedback{display:none;margin-top:.25rem;font-size:.875rem;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;width:250px;padding:.5rem;margin-top:.1rem;font-size:.875rem;line-height:1;color:#fff;background-color:rgba(220,53,69,.8);border-radius:.2rem}.custom-select.is-valid,.form-control.is-valid,.was-validated .custom-select:valid,.was-validated .form-control:valid{border-color:#28a745}.custom-select.is-valid:focus,.form-control.is-valid:focus,.was-validated .custom-select:valid:focus,.was-validated .form-control:valid:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-select.is-valid~.invalid-feedback,.custom-select.is-valid~.invalid-tooltip,.form-control.is-valid~.invalid-feedback,.form-control.is-valid~.invalid-tooltip,.was-validated .custom-select:valid~.invalid-feedback,.was-validated .custom-select:valid~.invalid-tooltip,.was-validated .form-control:valid~.invalid-feedback,.was-validated .form-control:valid~.invalid-tooltip{display:block}.form-check-input.is-valid+.form-check-label,.was-validated .form-check-input:valid+.form-check-label{color:#28a745}.custom-control-input.is-valid~.custom-control-indicator,.was-validated .custom-control-input:valid~.custom-control-indicator{background-color:rgba(40,167,69,.25)}.custom-control-input.is-valid~.custom-control-description,.was-validated .custom-control-input:valid~.custom-control-description{color:#28a745}.custom-file-input.is-valid~.custom-file-control,.was-validated .custom-file-input:valid~.custom-file-control{border-color:#28a745}.custom-file-input.is-valid~.custom-file-control::before,.was-validated .custom-file-input:valid~.custom-file-control::before{border-color:inherit}.custom-file-input.is-valid:focus,.was-validated .custom-file-input:valid:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-select.is-invalid,.form-control.is-invalid,.was-validated .custom-select:invalid,.was-validated .form-control:invalid{border-color:#dc3545}.custom-select.is-invalid:focus,.form-control.is-invalid:focus,.was-validated .custom-select:invalid:focus,.was-validated .form-control:invalid:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-select.is-invalid~.invalid-feedback,.custom-select.is-invalid~.invalid-tooltip,.form-control.is-invalid~.invalid-feedback,.form-control.is-invalid~.invalid-tooltip,.was-validated .custom-select:invalid~.invalid-feedback,.was-validated .custom-select:invalid~.invalid-tooltip,.was-validated .form-control:invalid~.invalid-feedback,.was-validated .form-control:invalid~.invalid-tooltip{display:block}.form-check-input.is-invalid+.form-check-label,.was-validated .form-check-input:invalid+.form-check-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-indicator,.was-validated .custom-control-input:invalid~.custom-control-indicator{background-color:rgba(220,53,69,.25)}.custom-control-input.is-invalid~.custom-control-description,.was-validated .custom-control-input:invalid~.custom-control-description{color:#dc3545}.custom-file-input.is-invalid~.custom-file-control,.was-validated .custom-file-input:invalid~.custom-file-control{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-control::before,.was-validated .custom-file-input:invalid~.custom-file-control::before{border-color:inherit}.custom-file-input.is-invalid:focus,.was-validated .custom-file-input:invalid:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .input-group{width:auto}.form-inline .form-control-label{margin-bottom:0;vertical-align:middle}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;margin-top:0;margin-bottom:0}.form-inline .form-check-label{padding-left:0}.form-inline .form-check-input{position:relative;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;padding-left:0}.form-inline .custom-control-indicator{position:static;display:inline-block;margin-right:.25rem;vertical-align:text-bottom}.form-inline .has-feedback .form-control-feedback{top:0}}.btn{display:inline-block;font-weight:400;text-align:center;white-space:nowrap;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;border:1px solid transparent;padding:.5rem .75rem;font-size:1rem;line-height:1.25;border-radius:.25rem;transition:all .15s ease-in-out}.btn:focus,.btn:hover{text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 3px rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}.btn.active,.btn:active{background-image:none}a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{box-shadow:0 0 0 3px rgba(0,123,255,.5)}.btn-primary.disabled,.btn-primary:disabled{background-color:#007bff;border-color:#007bff}.btn-primary.active,.btn-primary:active,.show>.btn-primary.dropdown-toggle{background-color:#0069d9;background-image:none;border-color:#0062cc}.btn-secondary{color:#fff;background-color:#868e96;border-color:#868e96}.btn-secondary:hover{color:#fff;background-color:#727b84;border-color:#6c757d}.btn-secondary.focus,.btn-secondary:focus{box-shadow:0 0 0 3px rgba(134,142,150,.5)}.btn-secondary.disabled,.btn-secondary:disabled{background-color:#868e96;border-color:#868e96}.btn-secondary.active,.btn-secondary:active,.show>.btn-secondary.dropdown-toggle{background-color:#727b84;background-image:none;border-color:#6c757d}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{box-shadow:0 0 0 3px rgba(40,167,69,.5)}.btn-success.disabled,.btn-success:disabled{background-color:#28a745;border-color:#28a745}.btn-success.active,.btn-success:active,.show>.btn-success.dropdown-toggle{background-color:#218838;background-image:none;border-color:#1e7e34}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{box-shadow:0 0 0 3px rgba(23,162,184,.5)}.btn-info.disabled,.btn-info:disabled{background-color:#17a2b8;border-color:#17a2b8}.btn-info.active,.btn-info:active,.show>.btn-info.dropdown-toggle{background-color:#138496;background-image:none;border-color:#117a8b}.btn-warning{color:#111;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#111;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{box-shadow:0 0 0 3px rgba(255,193,7,.5)}.btn-warning.disabled,.btn-warning:disabled{background-color:#ffc107;border-color:#ffc107}.btn-warning.active,.btn-warning:active,.show>.btn-warning.dropdown-toggle{background-color:#e0a800;background-image:none;border-color:#d39e00}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{box-shadow:0 0 0 3px rgba(220,53,69,.5)}.btn-danger.disabled,.btn-danger:disabled{background-color:#dc3545;border-color:#dc3545}.btn-danger.active,.btn-danger:active,.show>.btn-danger.dropdown-toggle{background-color:#c82333;background-image:none;border-color:#bd2130}.btn-light{color:#111;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#111;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{box-shadow:0 0 0 3px rgba(248,249,250,.5)}.btn-light.disabled,.btn-light:disabled{background-color:#f8f9fa;border-color:#f8f9fa}.btn-light.active,.btn-light:active,.show>.btn-light.dropdown-toggle{background-color:#e2e6ea;background-image:none;border-color:#dae0e5}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{box-shadow:0 0 0 3px rgba(52,58,64,.5)}.btn-dark.disabled,.btn-dark:disabled{background-color:#343a40;border-color:#343a40}.btn-dark.active,.btn-dark:active,.show>.btn-dark.dropdown-toggle{background-color:#23272b;background-image:none;border-color:#1d2124}.btn-outline-primary{color:#007bff;background-color:transparent;background-image:none;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 3px rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary.active,.btn-outline-primary:active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-secondary{color:#868e96;background-color:transparent;background-image:none;border-color:#868e96}.btn-outline-secondary:hover{color:#fff;background-color:#868e96;border-color:#868e96}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 3px rgba(134,142,150,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#868e96;background-color:transparent}.btn-outline-secondary.active,.btn-outline-secondary:active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#868e96;border-color:#868e96}.btn-outline-success{color:#28a745;background-color:transparent;background-image:none;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 3px rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success.active,.btn-outline-success:active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-info{color:#17a2b8;background-color:transparent;background-image:none;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 3px rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info.active,.btn-outline-info:active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-warning{color:#ffc107;background-color:transparent;background-image:none;border-color:#ffc107}.btn-outline-warning:hover{color:#fff;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 3px rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning.active,.btn-outline-warning:active,.show>.btn-outline-warning.dropdown-toggle{color:#fff;background-color:#ffc107;border-color:#ffc107}.btn-outline-danger{color:#dc3545;background-color:transparent;background-image:none;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 3px rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger.active,.btn-outline-danger:active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-light{color:#f8f9fa;background-color:transparent;background-image:none;border-color:#f8f9fa}.btn-outline-light:hover{color:#fff;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 3px rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light.active,.btn-outline-light:active,.show>.btn-outline-light.dropdown-toggle{color:#fff;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-dark{color:#343a40;background-color:transparent;background-image:none;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 3px rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark.active,.btn-outline-dark:active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-link{font-weight:400;color:#007bff;border-radius:0}.btn-link,.btn-link.active,.btn-link:active,.btn-link:disabled{background-color:transparent}.btn-link,.btn-link:active,.btn-link:focus{border-color:transparent;box-shadow:none}.btn-link:hover{border-color:transparent}.btn-link:focus,.btn-link:hover{color:#0056b3;text-decoration:underline;background-color:transparent}.btn-link:disabled{color:#868e96}.btn-link:disabled:focus,.btn-link:disabled:hover{text-decoration:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{opacity:0;transition:opacity .15s linear}.fade.show{opacity:1}.collapse{display:none}.collapse.show{display:block}tr.collapse.show{display:table-row}tbody.collapse.show{display:table-row-group}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}.dropdown,.dropup{position:relative}.dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropup .dropdown-menu{margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{border-top:0;border-bottom:.3em solid}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background:0 0;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#868e96;background-color:transparent}.show>a{outline:0}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#868e96;white-space:nowrap}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:0 1 auto;flex:0 1 auto;margin-bottom:0}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:2}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:2}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group,.btn-group-vertical .btn+.btn,.btn-group-vertical .btn+.btn-group,.btn-group-vertical .btn-group+.btn,.btn-group-vertical .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.btn-group>.btn-group{float:left}.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-bottom-left-radius:0}.btn+.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.btn+.dropdown-toggle-split::after{margin-left:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical .btn,.btn-group-vertical .btn-group{width:100%}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0}.btn-group-vertical>.btn:first-child:not(:last-child){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:last-child:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0}.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-top-right-radius:0}[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;width:100%}.input-group .form-control{position:relative;z-index:2;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;margin-bottom:0}.input-group .form-control:active,.input-group .form-control:focus,.input-group .form-control:hover{z-index:3}.input-group .form-control,.input-group-addon,.input-group-btn{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0}.input-group-addon,.input-group-btn{white-space:nowrap;vertical-align:middle}.input-group-addon{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.25;color:#495057;text-align:center;background-color:#e9ecef;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.input-group-addon.form-control-sm,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.input-group-addon.btn{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-addon.form-control-lg,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.input-group-addon.btn{padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0}.input-group .form-control:not(:last-child),.input-group-addon:not(:last-child),.input-group-btn:not(:first-child)>.btn-group:not(:last-child)>.btn,.input-group-btn:not(:first-child)>.btn:not(:last-child):not(.dropdown-toggle),.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group>.btn,.input-group-btn:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0}.input-group-addon:not(:last-child){border-right:0}.input-group .form-control:not(:first-child),.input-group-addon:not(:first-child),.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group>.btn,.input-group-btn:not(:first-child)>.dropdown-toggle,.input-group-btn:not(:last-child)>.btn-group:not(:first-child)>.btn,.input-group-btn:not(:last-child)>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.form-control+.input-group-addon:not(:first-child){border-left:0}.input-group-btn{position:relative;font-size:0;white-space:nowrap}.input-group-btn>.btn{position:relative}.input-group-btn>.btn+.btn{margin-left:-1px}.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:3}.input-group-btn:not(:last-child)>.btn,.input-group-btn:not(:last-child)>.btn-group{margin-right:-1px}.input-group-btn:not(:first-child)>.btn,.input-group-btn:not(:first-child)>.btn-group{z-index:2;margin-left:-1px}.input-group-btn:not(:first-child)>.btn-group:active,.input-group-btn:not(:first-child)>.btn-group:focus,.input-group-btn:not(:first-child)>.btn-group:hover,.input-group-btn:not(:first-child)>.btn:active,.input-group-btn:not(:first-child)>.btn:focus,.input-group-btn:not(:first-child)>.btn:hover{z-index:3}.custom-control{position:relative;display:-ms-inline-flexbox;display:inline-flex;min-height:1.5rem;padding-left:1.5rem;margin-right:1rem}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked~.custom-control-indicator{color:#fff;background-color:#007bff}.custom-control-input:focus~.custom-control-indicator{box-shadow:0 0 0 1px #fff,0 0 0 3px #007bff}.custom-control-input:active~.custom-control-indicator{color:#fff;background-color:#b3d7ff}.custom-control-input:disabled~.custom-control-indicator{background-color:#e9ecef}.custom-control-input:disabled~.custom-control-description{color:#868e96}.custom-control-indicator{position:absolute;top:.25rem;left:0;display:block;width:1rem;height:1rem;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#ddd;background-repeat:no-repeat;background-position:center center;background-size:50% 50%}.custom-checkbox .custom-control-indicator{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3E%3C/svg%3E")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-indicator{background-color:#007bff;background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3E%3Cpath stroke='%23fff' d='M0 2h4'/%3E%3C/svg%3E")}.custom-radio .custom-control-indicator{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-indicator{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3E%3Ccircle r='3' fill='%23fff'/%3E%3C/svg%3E")}.custom-controls-stacked{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column}.custom-controls-stacked .custom-control{margin-bottom:.25rem}.custom-controls-stacked .custom-control+.custom-control{margin-left:0}.custom-select{display:inline-block;max-width:100%;height:calc(2.25rem + 2px);padding:.375rem 1.75rem .375rem .75rem;line-height:1.25;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3E%3Cpath fill='%23333' d='M2 0L0 2h4zm0 5L0 3h4z'/%3E%3C/svg%3E") no-repeat right .75rem center;background-size:8px 10px;border:1px solid rgba(0,0,0,.15);border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select:disabled{color:#868e96;background-color:#e9ecef}.custom-select::-ms-expand{opacity:0}.custom-select-sm{height:calc(1.8125rem + 2px);padding-top:.375rem;padding-bottom:.375rem;font-size:75%}.custom-file{position:relative;display:inline-block;max-width:100%;height:2.5rem;margin-bottom:0}.custom-file-input{min-width:14rem;max-width:100%;height:2.5rem;margin:0;opacity:0}.custom-file-control{position:absolute;top:0;right:0;left:0;z-index:5;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#495057;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#fff;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.custom-file-control:lang(en):empty::after{content:"Choose file..."}.custom-file-control::before{position:absolute;top:-1px;right:-1px;bottom:-1px;z-index:6;display:block;height:2.5rem;padding:.5rem 1rem;line-height:1.5;color:#495057;background-color:#e9ecef;border:1px solid rgba(0,0,0,.15);border-radius:0 .25rem .25rem 0}.custom-file-control:lang(en)::before{content:"Browse"}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#868e96}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #ddd}.nav-tabs .nav-link.disabled{color:#868e96;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#ddd #ddd #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.show>.nav-pills .nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar>.container,.navbar>.container-fluid{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background:0 0;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .dropdown-menu-right{right:0;left:auto}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .dropdown-menu-right{right:0;left:auto}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .dropdown-menu-right{right:0;left:auto}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .dropdown-menu-right{right:0;left:auto}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:nowrap;flex-wrap:nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .dropdown-menu-right{right:0;left:auto}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(0, 0, 0, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(255, 255, 255, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card>.list-group:first-child .list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img{width:100%;border-radius:calc(.25rem - 1px)}.card-img-top{width:100%;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img-bottom{width:100%;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{display:-ms-flexbox;display:flex;-ms-flex:1 0 0%;flex:1 0 0%;-ms-flex-direction:column;flex-direction:column;margin-right:15px;margin-left:15px}}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group .card{-ms-flex:1 0 0%;flex:1 0 0%}.card-group .card+.card{margin-left:0;border-left:0}.card-group .card:first-child{border-top-right-radius:0;border-bottom-right-radius:0}.card-group .card:first-child .card-img-top{border-top-right-radius:0}.card-group .card:first-child .card-img-bottom{border-bottom-right-radius:0}.card-group .card:last-child{border-top-left-radius:0;border-bottom-left-radius:0}.card-group .card:last-child .card-img-top{border-top-left-radius:0}.card-group .card:last-child .card-img-bottom{border-bottom-left-radius:0}.card-group .card:not(:first-child):not(:last-child){border-radius:0}.card-group .card:not(:first-child):not(:last-child) .card-img-bottom,.card-group .card:not(:first-child):not(:last-child) .card-img-top{border-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;column-count:3;-webkit-column-gap:1.25rem;column-gap:1.25rem}.card-columns .card{display:inline-block;width:100%}}.breadcrumb{padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb::after{display:block;clear:both;content:""}.breadcrumb-item{float:left}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;padding-left:.5rem;color:#868e96;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#868e96}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#868e96;pointer-events:none;background-color:#fff;border-color:#ddd}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #ddd}.page-link:focus,.page-link:hover{color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#ddd}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}.badge-primary[href]:focus,.badge-primary[href]:hover{color:#fff;text-decoration:none;background-color:#0062cc}.badge-secondary{color:#fff;background-color:#868e96}.badge-secondary[href]:focus,.badge-secondary[href]:hover{color:#fff;text-decoration:none;background-color:#6c757d}.badge-success{color:#fff;background-color:#28a745}.badge-success[href]:focus,.badge-success[href]:hover{color:#fff;text-decoration:none;background-color:#1e7e34}.badge-info{color:#fff;background-color:#17a2b8}.badge-info[href]:focus,.badge-info[href]:hover{color:#fff;text-decoration:none;background-color:#117a8b}.badge-warning{color:#111;background-color:#ffc107}.badge-warning[href]:focus,.badge-warning[href]:hover{color:#111;text-decoration:none;background-color:#d39e00}.badge-danger{color:#fff;background-color:#dc3545}.badge-danger[href]:focus,.badge-danger[href]:hover{color:#fff;text-decoration:none;background-color:#bd2130}.badge-light{color:#111;background-color:#f8f9fa}.badge-light[href]:focus,.badge-light[href]:hover{color:#111;text-decoration:none;background-color:#dae0e5}.badge-dark{color:#fff;background-color:#343a40}.badge-dark[href]:focus,.badge-dark[href]:hover{color:#fff;text-decoration:none;background-color:#1d2124}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible .close{position:relative;top:-.75rem;right:-1.25rem;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#464a4e;background-color:#e7e8ea;border-color:#dddfe2}.alert-secondary hr{border-top-color:#cfd2d6}.alert-secondary .alert-link{color:#2e3133}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;overflow:hidden;font-size:.75rem;line-height:1rem;text-align:center;background-color:#e9ecef;border-radius:.25rem}.progress-bar{height:1rem;line-height:1rem;color:#fff;background-color:#007bff;transition:width .6s ease}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.list-group-item:focus,.list-group-item:hover{text-decoration:none}.list-group-item.disabled,.list-group-item:disabled{color:#868e96;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{border-bottom:0}.list-group-item-primary{color:#004085;background-color:#b8daff}a.list-group-item-primary,button.list-group-item-primary{color:#004085}a.list-group-item-primary:focus,a.list-group-item-primary:hover,button.list-group-item-primary:focus,button.list-group-item-primary:hover{color:#004085;background-color:#9fcdff}a.list-group-item-primary.active,button.list-group-item-primary.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#464a4e;background-color:#dddfe2}a.list-group-item-secondary,button.list-group-item-secondary{color:#464a4e}a.list-group-item-secondary:focus,a.list-group-item-secondary:hover,button.list-group-item-secondary:focus,button.list-group-item-secondary:hover{color:#464a4e;background-color:#cfd2d6}a.list-group-item-secondary.active,button.list-group-item-secondary.active{color:#fff;background-color:#464a4e;border-color:#464a4e}.list-group-item-success{color:#155724;background-color:#c3e6cb}a.list-group-item-success,button.list-group-item-success{color:#155724}a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#155724;background-color:#b1dfbb}a.list-group-item-success.active,button.list-group-item-success.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}a.list-group-item-info,button.list-group-item-info{color:#0c5460}a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#0c5460;background-color:#abdde5}a.list-group-item-info.active,button.list-group-item-info.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}a.list-group-item-warning,button.list-group-item-warning{color:#856404}a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#856404;background-color:#ffe8a1}a.list-group-item-warning.active,button.list-group-item-warning.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}a.list-group-item-danger,button.list-group-item-danger{color:#721c24}a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#721c24;background-color:#f1b0b7}a.list-group-item-danger.active,button.list-group-item-danger.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}a.list-group-item-light,button.list-group-item-light{color:#818182}a.list-group-item-light:focus,a.list-group-item-light:hover,button.list-group-item-light:focus,button.list-group-item-light:hover{color:#818182;background-color:#ececf6}a.list-group-item-light.active,button.list-group-item-light.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}a.list-group-item-dark,button.list-group-item-dark{color:#1b1e21}a.list-group-item-dark:focus,a.list-group-item-dark:hover,button.list-group-item-dark:focus,button.list-group-item-dark:hover{color:#1b1e21;background-color:#b9bbbe}a.list-group-item-dark.active,button.list-group-item-dark.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:focus,.close:hover{color:#000;text-decoration:none;opacity:.75}button.close{padding:0;background:0 0;border:0;-webkit-appearance:none}.modal-open{overflow:hidden}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;outline:0}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-25%);transform:translate(0,-25%)}.modal.show .modal-dialog{-webkit-transform:translate(0,0);transform:translate(0,0)}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal-dialog{position:relative;width:auto;margin:10px}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:15px;border-bottom:1px solid #e9ecef}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:15px}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:15px;border-top:1px solid #e9ecef}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:30px auto}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg{max-width:800px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:5px;height:5px}.tooltip.bs-tooltip-auto[x-placement^=top],.tooltip.bs-tooltip-top{padding:5px 0}.tooltip.bs-tooltip-auto[x-placement^=top] .arrow,.tooltip.bs-tooltip-top .arrow{bottom:0}.tooltip.bs-tooltip-auto[x-placement^=top] .arrow::before,.tooltip.bs-tooltip-top .arrow::before{margin-left:-3px;content:"";border-width:5px 5px 0;border-top-color:#000}.tooltip.bs-tooltip-auto[x-placement^=right],.tooltip.bs-tooltip-right{padding:0 5px}.tooltip.bs-tooltip-auto[x-placement^=right] .arrow,.tooltip.bs-tooltip-right .arrow{left:0}.tooltip.bs-tooltip-auto[x-placement^=right] .arrow::before,.tooltip.bs-tooltip-right .arrow::before{margin-top:-3px;content:"";border-width:5px 5px 5px 0;border-right-color:#000}.tooltip.bs-tooltip-auto[x-placement^=bottom],.tooltip.bs-tooltip-bottom{padding:5px 0}.tooltip.bs-tooltip-auto[x-placement^=bottom] .arrow,.tooltip.bs-tooltip-bottom .arrow{top:0}.tooltip.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.tooltip.bs-tooltip-bottom .arrow::before{margin-left:-3px;content:"";border-width:0 5px 5px;border-bottom-color:#000}.tooltip.bs-tooltip-auto[x-placement^=left],.tooltip.bs-tooltip-left{padding:0 5px}.tooltip.bs-tooltip-auto[x-placement^=left] .arrow,.tooltip.bs-tooltip-left .arrow{right:0}.tooltip.bs-tooltip-auto[x-placement^=left] .arrow::before,.tooltip.bs-tooltip-left .arrow::before{right:0;margin-top:-3px;content:"";border-width:5px 0 5px 5px;border-left-color:#000}.tooltip .arrow::before{position:absolute;border-color:transparent;border-style:solid}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;padding:1px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:10px;height:5px}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;border-color:transparent;border-style:solid}.popover .arrow::before{content:"";border-width:11px}.popover .arrow::after{content:"";border-width:11px}.popover.bs-popover-auto[x-placement^=top],.popover.bs-popover-top{margin-bottom:10px}.popover.bs-popover-auto[x-placement^=top] .arrow,.popover.bs-popover-top .arrow{bottom:0}.popover.bs-popover-auto[x-placement^=top] .arrow::after,.popover.bs-popover-auto[x-placement^=top] .arrow::before,.popover.bs-popover-top .arrow::after,.popover.bs-popover-top .arrow::before{border-bottom-width:0}.popover.bs-popover-auto[x-placement^=top] .arrow::before,.popover.bs-popover-top .arrow::before{bottom:-11px;margin-left:-6px;border-top-color:rgba(0,0,0,.25)}.popover.bs-popover-auto[x-placement^=top] .arrow::after,.popover.bs-popover-top .arrow::after{bottom:-10px;margin-left:-6px;border-top-color:#fff}.popover.bs-popover-auto[x-placement^=right],.popover.bs-popover-right{margin-left:10px}.popover.bs-popover-auto[x-placement^=right] .arrow,.popover.bs-popover-right .arrow{left:0}.popover.bs-popover-auto[x-placement^=right] .arrow::after,.popover.bs-popover-auto[x-placement^=right] .arrow::before,.popover.bs-popover-right .arrow::after,.popover.bs-popover-right .arrow::before{margin-top:-8px;border-left-width:0}.popover.bs-popover-auto[x-placement^=right] .arrow::before,.popover.bs-popover-right .arrow::before{left:-11px;border-right-color:rgba(0,0,0,.25)}.popover.bs-popover-auto[x-placement^=right] .arrow::after,.popover.bs-popover-right .arrow::after{left:-10px;border-right-color:#fff}.popover.bs-popover-auto[x-placement^=bottom],.popover.bs-popover-bottom{margin-top:10px}.popover.bs-popover-auto[x-placement^=bottom] .arrow,.popover.bs-popover-bottom .arrow{top:0}.popover.bs-popover-auto[x-placement^=bottom] .arrow::after,.popover.bs-popover-auto[x-placement^=bottom] .arrow::before,.popover.bs-popover-bottom .arrow::after,.popover.bs-popover-bottom .arrow::before{margin-left:-7px;border-top-width:0}.popover.bs-popover-auto[x-placement^=bottom] .arrow::before,.popover.bs-popover-bottom .arrow::before{top:-11px;border-bottom-color:rgba(0,0,0,.25)}.popover.bs-popover-auto[x-placement^=bottom] .arrow::after,.popover.bs-popover-bottom .arrow::after{top:-10px;border-bottom-color:#fff}.popover.bs-popover-auto[x-placement^=bottom] .popover-header::before,.popover.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:20px;margin-left:-10px;content:"";border-bottom:1px solid #f7f7f7}.popover.bs-popover-auto[x-placement^=left],.popover.bs-popover-left{margin-right:10px}.popover.bs-popover-auto[x-placement^=left] .arrow,.popover.bs-popover-left .arrow{right:0}.popover.bs-popover-auto[x-placement^=left] .arrow::after,.popover.bs-popover-auto[x-placement^=left] .arrow::before,.popover.bs-popover-left .arrow::after,.popover.bs-popover-left .arrow::before{margin-top:-8px;border-right-width:0}.popover.bs-popover-auto[x-placement^=left] .arrow::before,.popover.bs-popover-left .arrow::before{right:-11px;border-left-color:rgba(0,0,0,.25)}.popover.bs-popover-auto[x-placement^=left] .arrow::after,.popover.bs-popover-left .arrow::after{right:-10px;border-left-color:#fff}.popover-header{padding:8px 14px;margin-bottom:0;font-size:1rem;color:inherit;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:9px 14px;color:#212529}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-item{position:relative;display:none;-ms-flex-align:center;align-items:center;width:100%;transition:-webkit-transform .6s ease;transition:transform .6s ease;transition:transform .6s ease,-webkit-transform .6s ease;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.carousel-item-next,.carousel-item-prev{position:absolute;top:0}.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translateX(0);transform:translateX(0)}@supports ((-webkit-transform-style:preserve-3d) or (transform-style:preserve-3d)){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}.active.carousel-item-right,.carousel-item-next{-webkit-transform:translateX(100%);transform:translateX(100%)}@supports ((-webkit-transform-style:preserve-3d) or (transform-style:preserve-3d)){.active.carousel-item-right,.carousel-item-next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}}.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translateX(-100%);transform:translateX(-100%)}@supports ((-webkit-transform-style:preserve-3d) or (transform-style:preserve-3d)){.active.carousel-item-left,.carousel-item-prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:transparent no-repeat center center;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M4 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3E%3C/svg%3E")}.carousel-control-next-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M1.5 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3E%3C/svg%3E")}.carousel-indicators{position:absolute;right:0;bottom:10px;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{position:relative;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;background-color:rgba(255,255,255,.5)}.carousel-indicators li::before{position:absolute;top:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators li::after{position:absolute;bottom:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators .active{background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#868e96!important}a.bg-secondary:focus,a.bg-secondary:hover{background-color:#6c757d!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #e9ecef!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#868e96!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-circle{border-radius:50%}.rounded-0{border-radius:0}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.d-print-block{display:none!important}@media print{.d-print-block{display:block!important}}.d-print-inline{display:none!important}@media print{.d-print-inline{display:inline!important}}.d-print-inline-block{display:none!important}@media print{.d-print-inline-block{display:inline-block!important}}@media print{.d-print-none{display:none!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;-webkit-clip-path:inset(50%);clip-path:inset(50%);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal;-webkit-clip-path:none;clip-path:none}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.m-0{margin:0!important}.mt-0{margin-top:0!important}.mr-0{margin-right:0!important}.mb-0{margin-bottom:0!important}.ml-0{margin-left:0!important}.mx-0{margin-right:0!important;margin-left:0!important}.my-0{margin-top:0!important;margin-bottom:0!important}.m-1{margin:.25rem!important}.mt-1{margin-top:.25rem!important}.mr-1{margin-right:.25rem!important}.mb-1{margin-bottom:.25rem!important}.ml-1{margin-left:.25rem!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-2{margin:.5rem!important}.mt-2{margin-top:.5rem!important}.mr-2{margin-right:.5rem!important}.mb-2{margin-bottom:.5rem!important}.ml-2{margin-left:.5rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-3{margin:1rem!important}.mt-3{margin-top:1rem!important}.mr-3{margin-right:1rem!important}.mb-3{margin-bottom:1rem!important}.ml-3{margin-left:1rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-4{margin:1.5rem!important}.mt-4{margin-top:1.5rem!important}.mr-4{margin-right:1.5rem!important}.mb-4{margin-bottom:1.5rem!important}.ml-4{margin-left:1.5rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-5{margin:3rem!important}.mt-5{margin-top:3rem!important}.mr-5{margin-right:3rem!important}.mb-5{margin-bottom:3rem!important}.ml-5{margin-left:3rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-0{padding:0!important}.pt-0{padding-top:0!important}.pr-0{padding-right:0!important}.pb-0{padding-bottom:0!important}.pl-0{padding-left:0!important}.px-0{padding-right:0!important;padding-left:0!important}.py-0{padding-top:0!important;padding-bottom:0!important}.p-1{padding:.25rem!important}.pt-1{padding-top:.25rem!important}.pr-1{padding-right:.25rem!important}.pb-1{padding-bottom:.25rem!important}.pl-1{padding-left:.25rem!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-2{padding:.5rem!important}.pt-2{padding-top:.5rem!important}.pr-2{padding-right:.5rem!important}.pb-2{padding-bottom:.5rem!important}.pl-2{padding-left:.5rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-3{padding:1rem!important}.pt-3{padding-top:1rem!important}.pr-3{padding-right:1rem!important}.pb-3{padding-bottom:1rem!important}.pl-3{padding-left:1rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-4{padding:1.5rem!important}.pt-4{padding-top:1.5rem!important}.pr-4{padding-right:1.5rem!important}.pb-4{padding-bottom:1.5rem!important}.pl-4{padding-left:1.5rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-5{padding:3rem!important}.pt-5{padding-top:3rem!important}.pr-5{padding-right:3rem!important}.pb-5{padding-bottom:3rem!important}.pl-5{padding-left:3rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-auto{margin:auto!important}.mt-auto{margin-top:auto!important}.mr-auto{margin-right:auto!important}.mb-auto{margin-bottom:auto!important}.ml-auto{margin-left:auto!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0{margin-top:0!important}.mr-sm-0{margin-right:0!important}.mb-sm-0{margin-bottom:0!important}.ml-sm-0{margin-left:0!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1{margin-top:.25rem!important}.mr-sm-1{margin-right:.25rem!important}.mb-sm-1{margin-bottom:.25rem!important}.ml-sm-1{margin-left:.25rem!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2{margin-top:.5rem!important}.mr-sm-2{margin-right:.5rem!important}.mb-sm-2{margin-bottom:.5rem!important}.ml-sm-2{margin-left:.5rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3{margin-top:1rem!important}.mr-sm-3{margin-right:1rem!important}.mb-sm-3{margin-bottom:1rem!important}.ml-sm-3{margin-left:1rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4{margin-top:1.5rem!important}.mr-sm-4{margin-right:1.5rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.ml-sm-4{margin-left:1.5rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5{margin-top:3rem!important}.mr-sm-5{margin-right:3rem!important}.mb-sm-5{margin-bottom:3rem!important}.ml-sm-5{margin-left:3rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0{padding-top:0!important}.pr-sm-0{padding-right:0!important}.pb-sm-0{padding-bottom:0!important}.pl-sm-0{padding-left:0!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1{padding-top:.25rem!important}.pr-sm-1{padding-right:.25rem!important}.pb-sm-1{padding-bottom:.25rem!important}.pl-sm-1{padding-left:.25rem!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2{padding-top:.5rem!important}.pr-sm-2{padding-right:.5rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pl-sm-2{padding-left:.5rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3{padding-top:1rem!important}.pr-sm-3{padding-right:1rem!important}.pb-sm-3{padding-bottom:1rem!important}.pl-sm-3{padding-left:1rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4{padding-top:1.5rem!important}.pr-sm-4{padding-right:1.5rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pl-sm-4{padding-left:1.5rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5{padding-top:3rem!important}.pr-sm-5{padding-right:3rem!important}.pb-sm-5{padding-bottom:3rem!important}.pl-sm-5{padding-left:3rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto{margin-top:auto!important}.mr-sm-auto{margin-right:auto!important}.mb-sm-auto{margin-bottom:auto!important}.ml-sm-auto{margin-left:auto!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0{margin-top:0!important}.mr-md-0{margin-right:0!important}.mb-md-0{margin-bottom:0!important}.ml-md-0{margin-left:0!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.m-md-1{margin:.25rem!important}.mt-md-1{margin-top:.25rem!important}.mr-md-1{margin-right:.25rem!important}.mb-md-1{margin-bottom:.25rem!important}.ml-md-1{margin-left:.25rem!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2{margin-top:.5rem!important}.mr-md-2{margin-right:.5rem!important}.mb-md-2{margin-bottom:.5rem!important}.ml-md-2{margin-left:.5rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3{margin-top:1rem!important}.mr-md-3{margin-right:1rem!important}.mb-md-3{margin-bottom:1rem!important}.ml-md-3{margin-left:1rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4{margin-top:1.5rem!important}.mr-md-4{margin-right:1.5rem!important}.mb-md-4{margin-bottom:1.5rem!important}.ml-md-4{margin-left:1.5rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5{margin-top:3rem!important}.mr-md-5{margin-right:3rem!important}.mb-md-5{margin-bottom:3rem!important}.ml-md-5{margin-left:3rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-md-0{padding:0!important}.pt-md-0{padding-top:0!important}.pr-md-0{padding-right:0!important}.pb-md-0{padding-bottom:0!important}.pl-md-0{padding-left:0!important}.px-md-0{padding-right:0!important;padding-left:0!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.p-md-1{padding:.25rem!important}.pt-md-1{padding-top:.25rem!important}.pr-md-1{padding-right:.25rem!important}.pb-md-1{padding-bottom:.25rem!important}.pl-md-1{padding-left:.25rem!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2{padding-top:.5rem!important}.pr-md-2{padding-right:.5rem!important}.pb-md-2{padding-bottom:.5rem!important}.pl-md-2{padding-left:.5rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3{padding-top:1rem!important}.pr-md-3{padding-right:1rem!important}.pb-md-3{padding-bottom:1rem!important}.pl-md-3{padding-left:1rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4{padding-top:1.5rem!important}.pr-md-4{padding-right:1.5rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pl-md-4{padding-left:1.5rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5{padding-top:3rem!important}.pr-md-5{padding-right:3rem!important}.pb-md-5{padding-bottom:3rem!important}.pl-md-5{padding-left:3rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto{margin-top:auto!important}.mr-md-auto{margin-right:auto!important}.mb-md-auto{margin-bottom:auto!important}.ml-md-auto{margin-left:auto!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0{margin-top:0!important}.mr-lg-0{margin-right:0!important}.mb-lg-0{margin-bottom:0!important}.ml-lg-0{margin-left:0!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1{margin-top:.25rem!important}.mr-lg-1{margin-right:.25rem!important}.mb-lg-1{margin-bottom:.25rem!important}.ml-lg-1{margin-left:.25rem!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2{margin-top:.5rem!important}.mr-lg-2{margin-right:.5rem!important}.mb-lg-2{margin-bottom:.5rem!important}.ml-lg-2{margin-left:.5rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3{margin-top:1rem!important}.mr-lg-3{margin-right:1rem!important}.mb-lg-3{margin-bottom:1rem!important}.ml-lg-3{margin-left:1rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4{margin-top:1.5rem!important}.mr-lg-4{margin-right:1.5rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.ml-lg-4{margin-left:1.5rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5{margin-top:3rem!important}.mr-lg-5{margin-right:3rem!important}.mb-lg-5{margin-bottom:3rem!important}.ml-lg-5{margin-left:3rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0{padding-top:0!important}.pr-lg-0{padding-right:0!important}.pb-lg-0{padding-bottom:0!important}.pl-lg-0{padding-left:0!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1{padding-top:.25rem!important}.pr-lg-1{padding-right:.25rem!important}.pb-lg-1{padding-bottom:.25rem!important}.pl-lg-1{padding-left:.25rem!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2{padding-top:.5rem!important}.pr-lg-2{padding-right:.5rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pl-lg-2{padding-left:.5rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3{padding-top:1rem!important}.pr-lg-3{padding-right:1rem!important}.pb-lg-3{padding-bottom:1rem!important}.pl-lg-3{padding-left:1rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4{padding-top:1.5rem!important}.pr-lg-4{padding-right:1.5rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pl-lg-4{padding-left:1.5rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5{padding-top:3rem!important}.pr-lg-5{padding-right:3rem!important}.pb-lg-5{padding-bottom:3rem!important}.pl-lg-5{padding-left:3rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto{margin-top:auto!important}.mr-lg-auto{margin-right:auto!important}.mb-lg-auto{margin-bottom:auto!important}.ml-lg-auto{margin-left:auto!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0{margin-top:0!important}.mr-xl-0{margin-right:0!important}.mb-xl-0{margin-bottom:0!important}.ml-xl-0{margin-left:0!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1{margin-top:.25rem!important}.mr-xl-1{margin-right:.25rem!important}.mb-xl-1{margin-bottom:.25rem!important}.ml-xl-1{margin-left:.25rem!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2{margin-top:.5rem!important}.mr-xl-2{margin-right:.5rem!important}.mb-xl-2{margin-bottom:.5rem!important}.ml-xl-2{margin-left:.5rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3{margin-top:1rem!important}.mr-xl-3{margin-right:1rem!important}.mb-xl-3{margin-bottom:1rem!important}.ml-xl-3{margin-left:1rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4{margin-top:1.5rem!important}.mr-xl-4{margin-right:1.5rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.ml-xl-4{margin-left:1.5rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5{margin-top:3rem!important}.mr-xl-5{margin-right:3rem!important}.mb-xl-5{margin-bottom:3rem!important}.ml-xl-5{margin-left:3rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0{padding-top:0!important}.pr-xl-0{padding-right:0!important}.pb-xl-0{padding-bottom:0!important}.pl-xl-0{padding-left:0!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1{padding-top:.25rem!important}.pr-xl-1{padding-right:.25rem!important}.pb-xl-1{padding-bottom:.25rem!important}.pl-xl-1{padding-left:.25rem!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2{padding-top:.5rem!important}.pr-xl-2{padding-right:.5rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pl-xl-2{padding-left:.5rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3{padding-top:1rem!important}.pr-xl-3{padding-right:1rem!important}.pb-xl-3{padding-bottom:1rem!important}.pl-xl-3{padding-left:1rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4{padding-top:1.5rem!important}.pr-xl-4{padding-right:1.5rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pl-xl-4{padding-left:1.5rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5{padding-top:3rem!important}.pr-xl-5{padding-right:3rem!important}.pb-xl-5{padding-bottom:3rem!important}.pl-xl-5{padding-left:3rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto{margin-top:auto!important}.mr-xl-auto{margin-right:auto!important}.mb-xl-auto{margin-bottom:auto!important}.ml-xl-auto{margin-left:auto!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}}.text-justify{text-align:justify!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-normal{font-weight:400}.font-weight-bold{font-weight:700}.font-italic{font-style:italic}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0062cc!important}.text-secondary{color:#868e96!important}a.text-secondary:focus,a.text-secondary:hover{color:#6c757d!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#1e7e34!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#117a8b!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#d39e00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#bd2130!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#dae0e5!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#1d2124!important}.text-muted{color:#868e96!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.visible{visibility:visible!important}.invisible{visibility:hidden!important} /*# sourceMappingURL=bootstrap.min.css.map */ ================================================ FILE: kafka-ui-api/src/main/resources/static/static/css/signin.css ================================================ body { padding-top: 40px; padding-bottom: 40px; background-color: #eee; } .form-signin { max-width: 330px; padding: 15px; margin: 0 auto; } .form-signin .form-signin-heading, .form-signin .checkbox { margin-bottom: 10px; } .form-signin .checkbox { font-weight: 400; } .form-signin .form-control { position: relative; box-sizing: border-box; height: auto; padding: 10px; font-size: 16px; } .form-signin .form-control:focus { z-index: 2; } .form-signin input[type="email"] { margin-bottom: -1px; border-bottom-right-radius: 0; border-bottom-left-radius: 0; } .form-signin input[type="password"] { margin-bottom: 10px; border-top-left-radius: 0; border-top-right-radius: 0; } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.container.KafkaConnectContainer; import com.provectus.kafka.ui.container.KsqlDbContainer; import com.provectus.kafka.ui.container.SchemaRegistryContainer; import java.nio.file.Path; import java.util.List; import java.util.Properties; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.function.ThrowingConsumer; import org.junit.jupiter.api.io.TempDir; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.util.TestSocketUtils; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.Network; import org.testcontainers.utility.DockerImageName; @SpringBootTest @ActiveProfiles("test") @AutoConfigureWebTestClient(timeout = "60000") @ContextConfiguration(initializers = {AbstractIntegrationTest.Initializer.class}) public abstract class AbstractIntegrationTest { public static final String LOCAL = "local"; public static final String SECOND_LOCAL = "secondLocal"; private static final String CONFLUENT_PLATFORM_VERSION = "7.2.1"; // Append ".arm64" for a local run public static final KafkaContainer kafka = new KafkaContainer( DockerImageName.parse("confluentinc/cp-kafka").withTag(CONFLUENT_PLATFORM_VERSION)) .withNetwork(Network.SHARED); public static final SchemaRegistryContainer schemaRegistry = new SchemaRegistryContainer(CONFLUENT_PLATFORM_VERSION) .withKafka(kafka) .dependsOn(kafka); public static final KafkaConnectContainer kafkaConnect = new KafkaConnectContainer(CONFLUENT_PLATFORM_VERSION) .withKafka(kafka) .dependsOn(kafka) .dependsOn(schemaRegistry); protected static final KsqlDbContainer KSQL_DB = new KsqlDbContainer( DockerImageName.parse("confluentinc/cp-ksqldb-server") .withTag(CONFLUENT_PLATFORM_VERSION)) .withKafka(kafka); @TempDir public static Path tmpDir; static { kafka.start(); schemaRegistry.start(); kafkaConnect.start(); } public static class Initializer implements ApplicationContextInitializer { @Override public void initialize(@NotNull ConfigurableApplicationContext context) { System.setProperty("kafka.clusters.0.name", LOCAL); System.setProperty("kafka.clusters.0.bootstrapServers", kafka.getBootstrapServers()); // List unavailable hosts to verify failover System.setProperty("kafka.clusters.0.schemaRegistry", String.format("http://localhost:%1$s,http://localhost:%1$s,%2$s", TestSocketUtils.findAvailableTcpPort(), schemaRegistry.getUrl())); System.setProperty("kafka.clusters.0.kafkaConnect.0.name", "kafka-connect"); System.setProperty("kafka.clusters.0.kafkaConnect.0.userName", "kafka-connect"); System.setProperty("kafka.clusters.0.kafkaConnect.0.password", "kafka-connect"); System.setProperty("kafka.clusters.0.kafkaConnect.0.address", kafkaConnect.getTarget()); System.setProperty("kafka.clusters.0.kafkaConnect.1.name", "notavailable"); System.setProperty("kafka.clusters.0.kafkaConnect.1.address", "http://notavailable:6666"); System.setProperty("kafka.clusters.0.masking.0.type", "REPLACE"); System.setProperty("kafka.clusters.0.masking.0.replacement", "***"); System.setProperty("kafka.clusters.0.masking.0.topicValuesPattern", "masking-test-.*"); System.setProperty("kafka.clusters.0.audit.topicAuditEnabled", "true"); System.setProperty("kafka.clusters.0.audit.consoleAuditEnabled", "true"); System.setProperty("kafka.clusters.1.name", SECOND_LOCAL); System.setProperty("kafka.clusters.1.readOnly", "true"); System.setProperty("kafka.clusters.1.bootstrapServers", kafka.getBootstrapServers()); System.setProperty("kafka.clusters.1.schemaRegistry", schemaRegistry.getUrl()); System.setProperty("kafka.clusters.1.kafkaConnect.0.name", "kafka-connect"); System.setProperty("kafka.clusters.1.kafkaConnect.0.address", kafkaConnect.getTarget()); System.setProperty("dynamic.config.enabled", "true"); System.setProperty("config.related.uploads.dir", tmpDir.toString()); } } public static void createTopic(NewTopic topic) { withAdminClient(client -> client.createTopics(List.of(topic)).all().get()); } public static void deleteTopic(String topic) { withAdminClient(client -> client.deleteTopics(List.of(topic)).all().get()); } private static void withAdminClient(ThrowingConsumer consumer) { Properties properties = new Properties(); properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); try (var client = AdminClient.create(properties)) { try { consumer.accept(client); } catch (Throwable throwable) { throw new RuntimeException(throwable); } } } @Autowired protected ConfigurableApplicationContext applicationContext; } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/KafkaConnectServiceTests.java ================================================ package com.provectus.kafka.ui; import static java.util.function.Predicate.not; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigValidationResponseDTO; import com.provectus.kafka.ui.model.ConnectorPluginConfigValueDTO; import com.provectus.kafka.ui.model.ConnectorPluginDTO; import com.provectus.kafka.ui.model.ConnectorStateDTO; import com.provectus.kafka.ui.model.ConnectorStatusDTO; import com.provectus.kafka.ui.model.ConnectorTypeDTO; import com.provectus.kafka.ui.model.NewConnectorDTO; import com.provectus.kafka.ui.model.TaskIdDTO; import java.util.List; import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; import org.springframework.test.web.reactive.server.WebTestClient; @Slf4j public class KafkaConnectServiceTests extends AbstractIntegrationTest { private final String connectName = "kafka-connect"; private final String connectorName = UUID.randomUUID().toString(); private final Map config = Map.of( "name", connectorName, "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test", "test.password", "******" ); @Autowired private WebTestClient webTestClient; @BeforeEach public void setUp() { webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(new NewConnectorDTO() .name(connectorName) .config(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test", "test.password", "test-credentials" )) ) .exchange() .expectStatus().isOk(); } @AfterEach public void tearDown() { webTestClient.delete() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk(); } @Test public void shouldListAllConnectors() { webTestClient.get() .uri("/api/clusters/{clusterName}/connectors", LOCAL) .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@.name == '%s')]", connectorName)) .exists(); } @Test public void shouldFilterByNameConnectors() { webTestClient.get() .uri( "/api/clusters/{clusterName}/connectors?search={search}", LOCAL, connectorName.split("-")[1]) .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@.name == '%s')]", connectorName)) .exists(); } @Test public void shouldFilterByStatusConnectors() { webTestClient.get() .uri( "/api/clusters/{clusterName}/connectors?search={search}", LOCAL, "running") .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@.name == '%s')]", connectorName)) .exists(); } @Test public void shouldFilterByTypeConnectors() { webTestClient.get() .uri( "/api/clusters/{clusterName}/connectors?search={search}", LOCAL, "sink") .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@.name == '%s')]", connectorName)) .exists(); } @Test public void shouldNotFilterConnectors() { webTestClient.get() .uri( "/api/clusters/{clusterName}/connectors?search={search}", LOCAL, "something-else") .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@.name == '%s')]", connectorName)) .doesNotExist(); } @Test public void shouldListConnectors() { webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .exchange() .expectStatus().isOk() .expectBodyList(String.class) .contains(connectorName); } @Test public void shouldReturnNotFoundForNonExistingCluster() { webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", "nonExistingCluster", connectName) .exchange() .expectStatus().isNotFound(); } @Test public void shouldReturnNotFoundForNonExistingConnectName() { webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, "nonExistingConnect") .exchange() .expectStatus().isNotFound(); } @Test public void shouldRetrieveConnector() { ConnectorDTO expected = (ConnectorDTO) new ConnectorDTO() .connect(connectName) .status(new ConnectorStatusDTO() .state(ConnectorStateDTO.RUNNING) .workerId("kafka-connect:8083")) .tasks(List.of(new TaskIdDTO() .connector(connectorName) .task(0))) .type(ConnectorTypeDTO.SINK) .name(connectorName) .config(config); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}", LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk() .expectBody(ConnectorDTO.class) .value(connector -> assertEquals(expected, connector)); } @Test public void shouldUpdateConfig() { webTestClient.put() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .bodyValue(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "another-topic", "file", "/tmp/new" ) ) .exchange() .expectStatus().isOk(); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() { }) .isEqualTo(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "another-topic", "file", "/tmp/new", "name", connectorName )); } @Test public void shouldReturn400WhenConnectReturns400ForInvalidConfigCreate() { var connectorName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(Map.of( "name", connectorName, "config", Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "invalid number", "topics", "another-topic", "file", "/tmp/test" )) ) .exchange() .expectStatus().isBadRequest(); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@ == '%s')]", connectorName)) .doesNotExist(); } @Test public void shouldReturn400WhenConnectReturns500ForInvalidConfigCreate() { var connectorName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(Map.of( "name", connectorName, "config", Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector" )) ) .exchange() .expectStatus().isBadRequest(); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .exchange() .expectStatus().isOk() .expectBody() .jsonPath(String.format("$[?(@ == '%s')]", connectorName)) .doesNotExist(); } @Test public void shouldReturn400WhenConnectReturns400ForInvalidConfigUpdate() { webTestClient.put() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .bodyValue(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "invalid number", "topics", "another-topic", "file", "/tmp/test" ) ) .exchange() .expectStatus().isBadRequest(); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() { }) .isEqualTo(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test", "name", connectorName, "test.password", "******" )); } @Test public void shouldReturn400WhenConnectReturns500ForInvalidConfigUpdate() { webTestClient.put() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .bodyValue(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector" ) ) .exchange() .expectStatus().isBadRequest(); webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config", LOCAL, connectName, connectorName) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() { }) .isEqualTo(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test", "test.password", "******", "name", connectorName )); } @Test public void shouldRetrieveConnectorPlugins() { webTestClient.get() .uri("/api/clusters/{clusterName}/connects/{connectName}/plugins", LOCAL, connectName) .exchange() .expectStatus().isOk() .expectBodyList(ConnectorPluginDTO.class) .value(plugins -> assertThat(plugins.size()).isGreaterThan(0)); } @Test public void shouldSuccessfullyValidateConnectorPluginConfiguration() { var pluginName = "FileStreamSinkConnector"; var path = "/api/clusters/{clusterName}/connects/{connectName}/plugins/{pluginName}/config/validate"; webTestClient.put() .uri(path, LOCAL, connectName, pluginName) .bodyValue(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test", "name", connectorName ) ) .exchange() .expectStatus().isOk() .expectBody(ConnectorPluginConfigValidationResponseDTO.class) .value(response -> assertEquals(0, response.getErrorCount())); } @Test public void shouldValidateAndReturnErrorsOfConnectorPluginConfiguration() { var pluginName = "FileStreamSinkConnector"; var path = "/api/clusters/{clusterName}/connects/{connectName}/plugins/{pluginName}/config/validate"; webTestClient.put() .uri(path, LOCAL, connectName, pluginName) .bodyValue(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "0", "topics", "output-topic", "file", "/tmp/test", "name", connectorName ) ) .exchange() .expectStatus().isOk() .expectBody(ConnectorPluginConfigValidationResponseDTO.class) .value(response -> { assertEquals(1, response.getErrorCount()); var error = response.getConfigs().stream() .map(ConnectorPluginConfigDTO::getValue) .map(ConnectorPluginConfigValueDTO::getErrors) .filter(not(List::isEmpty)) .findFirst().get(); assertEquals( "Invalid value 0 for configuration tasks.max: Value must be at least 1", error.get(0) ); }); } @Test public void shouldReturn400WhenTryingToCreateConnectorWithExistingName() { webTestClient.post() .uri("/api/clusters/{clusterName}/connects/{connectName}/connectors", LOCAL, connectName) .bodyValue(new NewConnectorDTO() .name(connectorName) .config(Map.of( "connector.class", "org.apache.kafka.connect.file.FileStreamSinkConnector", "tasks.max", "1", "topics", "output-topic", "file", "/tmp/test" )) ) .exchange() .expectStatus() .isBadRequest(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/KafkaConsumerGroupTests.java ================================================ package com.provectus.kafka.ui; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.model.ConsumerGroupDTO; import com.provectus.kafka.ui.model.ConsumerGroupsPageResponseDTO; import java.io.Closeable; import java.time.Duration; import java.util.Comparator; import java.util.List; import java.util.Properties; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.RandomStringUtils; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.BytesDeserializer; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.reactive.server.WebTestClient; @Slf4j public class KafkaConsumerGroupTests extends AbstractIntegrationTest { @Autowired WebTestClient webTestClient; @Test void shouldNotFoundWhenNoSuchConsumerGroupId() { String groupId = "groupA"; String expError = "The group id does not exist"; webTestClient .delete() .uri("/api/clusters/{clusterName}/consumer-groups/{groupId}", LOCAL, groupId) .exchange() .expectStatus() .isNotFound(); } @Test void shouldOkWhenConsumerGroupIsNotActive() { String topicName = createTopicWithRandomName(); //Create a consumer and subscribe to the topic String groupId = UUID.randomUUID().toString(); val consumer = createTestConsumerWithGroupId(groupId); consumer.subscribe(List.of(topicName)); consumer.poll(Duration.ofMillis(100)); //Unsubscribe from all topics to be able to delete this consumer consumer.unsubscribe(); //Delete the consumer when it's INACTIVE and check webTestClient .delete() .uri("/api/clusters/{clusterName}/consumer-groups/{groupId}", LOCAL, groupId) .exchange() .expectStatus() .isOk(); } @Test void shouldBeBadRequestWhenConsumerGroupIsActive() { String topicName = createTopicWithRandomName(); //Create a consumer and subscribe to the topic String groupId = UUID.randomUUID().toString(); val consumer = createTestConsumerWithGroupId(groupId); consumer.subscribe(List.of(topicName)); consumer.poll(Duration.ofMillis(100)); //Try to delete the consumer when it's ACTIVE String expError = "The group is not empty"; webTestClient .delete() .uri("/api/clusters/{clusterName}/consumer-groups/{groupId}", LOCAL, groupId) .exchange() .expectStatus() .isBadRequest(); } @Test void shouldReturnConsumerGroupsWithPagination() throws Exception { try (var groups1 = startConsumerGroups(3, "cgPageTest1"); var groups2 = startConsumerGroups(2, "cgPageTest2")) { webTestClient .get() .uri("/api/clusters/{clusterName}/consumer-groups/paged?perPage=3&search=cgPageTest", LOCAL) .exchange() .expectStatus() .isOk() .expectBody(ConsumerGroupsPageResponseDTO.class) .value(page -> { assertThat(page.getPageCount()).isEqualTo(2); assertThat(page.getConsumerGroups().size()).isEqualTo(3); }); webTestClient .get() .uri("/api/clusters/{clusterName}/consumer-groups/paged?perPage=10&search=cgPageTest", LOCAL) .exchange() .expectStatus() .isOk() .expectBody(ConsumerGroupsPageResponseDTO.class) .value(page -> { assertThat(page.getPageCount()).isEqualTo(1); assertThat(page.getConsumerGroups().size()).isEqualTo(5); assertThat(page.getConsumerGroups()) .isSortedAccordingTo(Comparator.comparing(ConsumerGroupDTO::getGroupId)); }); webTestClient .get() .uri("/api/clusters/{clusterName}/consumer-groups/paged?perPage=10&&search" + "=cgPageTest&orderBy=NAME&sortOrder=DESC", LOCAL) .exchange() .expectStatus() .isOk() .expectBody(ConsumerGroupsPageResponseDTO.class) .value(page -> { assertThat(page.getPageCount()).isEqualTo(1); assertThat(page.getConsumerGroups().size()).isEqualTo(5); assertThat(page.getConsumerGroups()) .isSortedAccordingTo(Comparator.comparing(ConsumerGroupDTO::getGroupId).reversed()); }); webTestClient .get() .uri("/api/clusters/{clusterName}/consumer-groups/paged?perPage=10&&search" + "=cgPageTest&orderBy=MEMBERS&sortOrder=DESC", LOCAL) .exchange() .expectStatus() .isOk() .expectBody(ConsumerGroupsPageResponseDTO.class) .value(page -> { assertThat(page.getPageCount()).isEqualTo(1); assertThat(page.getConsumerGroups().size()).isEqualTo(5); assertThat(page.getConsumerGroups()) .isSortedAccordingTo(Comparator.comparing(ConsumerGroupDTO::getMembers).reversed()); }); } } private Closeable startConsumerGroups(int count, String consumerGroupPrefix) { String topicName = createTopicWithRandomName(); var consumers = Stream.generate(() -> { String groupId = consumerGroupPrefix + RandomStringUtils.randomAlphabetic(5); val consumer = createTestConsumerWithGroupId(groupId); consumer.subscribe(List.of(topicName)); consumer.poll(Duration.ofMillis(100)); return consumer; }) .limit(count) .collect(Collectors.toList()); return () -> { consumers.forEach(KafkaConsumer::close); deleteTopic(topicName); }; } private String createTopicWithRandomName() { String topicName = getClass().getSimpleName() + "-" + UUID.randomUUID(); short replicationFactor = 1; int partitions = 1; createTopic(new NewTopic(topicName, partitions, replicationFactor)); return topicName; } private KafkaConsumer createTestConsumerWithGroupId(String groupId) { Properties props = new Properties(); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); props.put(ConsumerConfig.CLIENT_ID_CONFIG, groupId); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); return new KafkaConsumer<>(props); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/KafkaConsumerTests.java ================================================ package com.provectus.kafka.ui; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.http.MediaType.TEXT_EVENT_STREAM; import com.provectus.kafka.ui.model.BrokerConfigDTO; import com.provectus.kafka.ui.model.PartitionsIncreaseDTO; import com.provectus.kafka.ui.model.PartitionsIncreaseResponseDTO; import com.provectus.kafka.ui.model.TopicConfigDTO; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.TopicDetailsDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.producer.KafkaTestProducer; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.reactive.server.WebTestClient; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @Slf4j public class KafkaConsumerTests extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @Test public void shouldDeleteRecords() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) .configs(Map.of()) ) .exchange() .expectStatus() .isOk(); try (KafkaTestProducer producer = KafkaTestProducer.forKafka(kafka)) { Flux.fromStream( Stream.of("one", "two", "three", "four") .map(value -> Mono.fromFuture(producer.send(topicName, value))) ).blockLast(); } catch (Throwable e) { log.error("Error on sending", e); throw new RuntimeException(e); } long count = webTestClient.get() .uri("/api/clusters/{clusterName}/topics/{topicName}/messages", LOCAL, topicName) .accept(TEXT_EVENT_STREAM) .exchange() .expectStatus() .isOk() .expectBodyList(TopicMessageEventDTO.class) .returnResult() .getResponseBody() .stream() .filter(e -> e.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) .count(); assertThat(count).isEqualTo(4); webTestClient.delete() .uri("/api/clusters/{clusterName}/topics/{topicName}/messages", LOCAL, topicName) .exchange() .expectStatus() .isOk(); count = webTestClient.get() .uri("/api/clusters/{clusterName}/topics/{topicName}/messages", LOCAL, topicName) .exchange() .expectStatus() .isOk() .expectBodyList(TopicMessageEventDTO.class) .returnResult() .getResponseBody() .stream() .filter(e -> e.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) .count(); assertThat(count).isZero(); } @Test public void shouldIncreasePartitionsUpTo10() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) .configs(Map.of()) ) .exchange() .expectStatus() .isOk(); PartitionsIncreaseResponseDTO response = webTestClient.patch() .uri("/api/clusters/{clusterName}/topics/{topicName}/partitions", LOCAL, topicName) .bodyValue(new PartitionsIncreaseDTO() .totalPartitionsCount(10) ) .exchange() .expectStatus() .isOk() .expectBody(PartitionsIncreaseResponseDTO.class) .returnResult() .getResponseBody(); assert response != null; Assertions.assertEquals(10, response.getTotalPartitionsCount()); TopicDetailsDTO topicDetails = webTestClient.get() .uri("/api/clusters/{clusterName}/topics/{topicName}", LOCAL, topicName) .exchange() .expectStatus() .isOk() .expectBody(TopicDetailsDTO.class) .returnResult() .getResponseBody(); assert topicDetails != null; Assertions.assertEquals(10, topicDetails.getPartitionCount()); } @Test public void shouldReturn404ForNonExistingTopic() { var topicName = UUID.randomUUID().toString(); webTestClient.delete() .uri("/api/clusters/{clusterName}/topics/{topicName}/messages", LOCAL, topicName) .exchange() .expectStatus() .isNotFound(); webTestClient.get() .uri("/api/clusters/{clusterName}/topics/{topicName}/config", LOCAL, topicName) .exchange() .expectStatus() .isNotFound(); } @Test public void shouldReturnConfigsForBroker() { var topicName = UUID.randomUUID().toString(); List configs = webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/{id}/configs", LOCAL, 1) .exchange() .expectStatus() .isOk() .expectBodyList(BrokerConfigDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(configs); assert !configs.isEmpty(); Assertions.assertNotNull(configs.get(0).getName()); Assertions.assertNotNull(configs.get(0).getIsReadOnly()); Assertions.assertNotNull(configs.get(0).getIsSensitive()); Assertions.assertNotNull(configs.get(0).getSource()); Assertions.assertNotNull(configs.get(0).getSynonyms()); } @Test public void shouldReturn404ForNonExistingBroker() { webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/{id}/configs", LOCAL, 0) .exchange() .expectStatus() .isNotFound(); } @Test public void shouldRetrieveTopicConfig() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) .configs(Map.of()) ) .exchange() .expectStatus() .isOk(); List configs = webTestClient.get() .uri("/api/clusters/{clusterName}/topics/{topicName}/config", LOCAL, topicName) .exchange() .expectStatus() .isOk() .expectBodyList(TopicConfigDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(configs); assert !configs.isEmpty(); Assertions.assertNotNull(configs.get(0).getName()); Assertions.assertNotNull(configs.get(0).getIsReadOnly()); Assertions.assertNotNull(configs.get(0).getIsSensitive()); Assertions.assertNotNull(configs.get(0).getSource()); Assertions.assertNotNull(configs.get(0).getSynonyms()); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/KafkaTopicCreateTests.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.model.TopicCreationDTO; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.reactive.server.WebTestClient; public class KafkaTopicCreateTests extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; private TopicCreationDTO topicCreation; @BeforeEach public void setUpBefore() { this.topicCreation = new TopicCreationDTO() .replicationFactor(1) .partitions(3) .name(UUID.randomUUID().toString()); } @Test void shouldCreateNewTopicSuccessfully() { webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(topicCreation) .exchange() .expectStatus() .isOk(); } @Test void shouldReturn400IfTopicAlreadyExists() { TopicCreationDTO topicCreation = new TopicCreationDTO() .replicationFactor(1) .partitions(3) .name(UUID.randomUUID().toString()); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(topicCreation) .exchange() .expectStatus() .isOk(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(topicCreation) .exchange() .expectStatus() .isBadRequest(); } @Test void shouldRecreateExistingTopicSuccessfully() { TopicCreationDTO topicCreation = new TopicCreationDTO() .replicationFactor(1) .partitions(3) .name(UUID.randomUUID().toString()); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(topicCreation) .exchange() .expectStatus() .isOk(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics/" + topicCreation.getName(), LOCAL) .exchange() .expectStatus() .isCreated() .expectBody() .jsonPath("partitionCount").isEqualTo(topicCreation.getPartitions().toString()) .jsonPath("replicationFactor").isEqualTo(topicCreation.getReplicationFactor().toString()) .jsonPath("name").isEqualTo(topicCreation.getName()); } @Test void shouldCloneExistingTopicSuccessfully() { TopicCreationDTO topicCreation = new TopicCreationDTO() .replicationFactor(1) .partitions(3) .name(UUID.randomUUID().toString()); String clonedTopicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(topicCreation) .exchange() .expectStatus() .isOk(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics/{topicName}/clone?newTopicName=" + clonedTopicName, LOCAL, topicCreation.getName()) .exchange() .expectStatus() .isCreated() .expectBody() .jsonPath("partitionCount").isEqualTo(topicCreation.getPartitions().toString()) .jsonPath("replicationFactor").isEqualTo(topicCreation.getReplicationFactor().toString()) .jsonPath("name").isEqualTo(clonedTopicName); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/ReadOnlyModeTests.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.TopicUpdateDTO; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.test.web.reactive.server.WebTestClient; public class ReadOnlyModeTests extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @Test public void shouldCreateTopicForNonReadonlyCluster() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) .configs(Map.of()) ) .exchange() .expectStatus() .isOk(); } @Test public void shouldNotCreateTopicForReadonlyCluster() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", SECOND_LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) .configs(Map.of()) ) .exchange() .expectStatus() .isEqualTo(HttpStatus.METHOD_NOT_ALLOWED); } @Test public void shouldUpdateTopicForNonReadonlyCluster() { var topicName = UUID.randomUUID().toString(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue(new TopicCreationDTO() .name(topicName) .partitions(1) .replicationFactor(1) ) .exchange() .expectStatus() .isOk(); webTestClient.patch() .uri("/api/clusters/{clusterName}/topics/{topicName}", LOCAL, topicName) .bodyValue(new TopicUpdateDTO() .configs(Map.of("cleanup.policy", "compact")) ) .exchange() .expectStatus() .isOk() .expectBody() .jsonPath("$.cleanUpPolicy").isEqualTo("COMPACT"); } @Test public void shouldNotUpdateTopicForReadonlyCluster() { var topicName = UUID.randomUUID().toString(); webTestClient.patch() .uri("/api/clusters/{clusterName}/topics/{topicName}", SECOND_LOCAL, topicName) .bodyValue(new TopicUpdateDTO() .configs(Map.of()) ) .exchange() .expectStatus() .isEqualTo(HttpStatus.METHOD_NOT_ALLOWED); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/SchemaRegistryServiceTests.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.model.CompatibilityLevelDTO; import com.provectus.kafka.ui.model.NewSchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaReferenceDTO; import com.provectus.kafka.ui.model.SchemaSubjectDTO; import com.provectus.kafka.ui.model.SchemaSubjectsResponseDTO; import com.provectus.kafka.ui.model.SchemaTypeDTO; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Objects; import java.util.UUID; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.test.web.reactive.server.EntityExchangeResult; import org.springframework.test.web.reactive.server.WebTestClient; import org.springframework.web.reactive.function.BodyInserters; import org.testcontainers.shaded.org.hamcrest.MatcherAssert; import org.testcontainers.shaded.org.hamcrest.Matchers; import reactor.core.publisher.Mono; @Slf4j class SchemaRegistryServiceTests extends AbstractIntegrationTest { @Autowired WebTestClient webTestClient; String subject; @BeforeEach public void setUpBefore() { this.subject = UUID.randomUUID().toString(); } @Test public void should404WhenGetAllSchemasForUnknownCluster() { webTestClient .get() .uri("/api/clusters/unknown-cluster/schemas") .exchange() .expectStatus().isNotFound(); } @Test public void shouldReturn404WhenGetLatestSchemaByNonExistingSubject() { String unknownSchema = "unknown-schema"; webTestClient .get() .uri("/api/clusters/{clusterName}/schemas/{subject}/latest", LOCAL, unknownSchema) .exchange() .expectStatus().isNotFound(); } /** * It should create a new schema w/o submitting a schemaType field to Schema Registry. */ @Test void shouldBeBadRequestIfNoSchemaType() { String schema = "{\"subject\":\"%s\",\"schema\":\"{\\\"type\\\": \\\"string\\\"}\"}"; webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(String.format(schema, subject))) .exchange() .expectStatus().isBadRequest(); } @Test void shouldNotDoAnythingIfSchemaNotChanged() { String schema = "{\"subject\":\"%s\",\"schemaType\":\"AVRO\",\"schema\":" + "\"{\\\"type\\\": \\\"string\\\"}\"}"; SchemaSubjectDTO dto = webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(String.format(schema, subject))) .exchange() .expectStatus() .isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(dto); Assertions.assertEquals("1", dto.getVersion()); dto = webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(String.format(schema, subject))) .exchange() .expectStatus() .isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(dto); Assertions.assertEquals("1", dto.getVersion()); } @Test void shouldReturnCorrectMessageWhenIncompatibleSchema() { String schema = "{\"subject\":\"%s\",\"schemaType\":\"JSON\",\"schema\":" + "\"{\\\"type\\\": \\\"string\\\"," + "\\\"properties\\\": " + "{\\\"f1\\\": {\\\"type\\\": \\\"integer\\\"}}}" + "\"}"; String schema2 = "{\"subject\":\"%s\"," + "\"schemaType\":\"JSON\",\"schema\":" + "\"{\\\"type\\\": \\\"string\\\"," + "\\\"properties\\\": " + "{\\\"f1\\\": {\\\"type\\\": \\\"string\\\"}," + "\\\"f2\\\": {" + "\\\"type\\\": \\\"string\\\"}}}" + "\"}"; SchemaSubjectDTO dto = webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(String.format(schema, subject))) .exchange() .expectStatus() .isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(dto); Assertions.assertEquals("1", dto.getVersion()); webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(String.format(schema2, subject))) .exchange() .expectStatus().isEqualTo(HttpStatus.UNPROCESSABLE_ENTITY) .expectBody().consumeWith(body -> { String responseBody = new String(Objects.requireNonNull(body.getResponseBody()), StandardCharsets.UTF_8); MatcherAssert.assertThat("Must return correct message incompatible schema", responseBody, Matchers.containsString("Schema being registered is incompatible with an earlier schema")); }); dto = webTestClient .get() .uri("/api/clusters/{clusterName}/schemas/{subject}/latest", LOCAL, subject) .exchange() .expectStatus().isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(dto); Assertions.assertEquals("1", dto.getVersion()); } @Test void shouldCreateNewProtobufSchema() { String schema = "syntax = \"proto3\";\n\nmessage MyRecord {\n int32 id = 1;\n string name = 2;\n}\n"; NewSchemaSubjectDTO requestBody = new NewSchemaSubjectDTO() .schemaType(SchemaTypeDTO.PROTOBUF) .subject(subject) .schema(schema); SchemaSubjectDTO actual = webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromPublisher(Mono.just(requestBody), NewSchemaSubjectDTO.class)) .exchange() .expectStatus() .isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(actual); Assertions.assertEquals(CompatibilityLevelDTO.CompatibilityEnum.BACKWARD.name(), actual.getCompatibilityLevel()); Assertions.assertEquals("1", actual.getVersion()); Assertions.assertEquals(SchemaTypeDTO.PROTOBUF, actual.getSchemaType()); Assertions.assertEquals(schema, actual.getSchema()); } @Test void shouldCreateNewProtobufSchemaWithRefs() { NewSchemaSubjectDTO requestBody = new NewSchemaSubjectDTO() .schemaType(SchemaTypeDTO.PROTOBUF) .subject(subject + "-ref") .schema(""" syntax = "proto3"; message MyRecord { int32 id = 1; string name = 2; } """); webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromPublisher(Mono.just(requestBody), NewSchemaSubjectDTO.class)) .exchange() .expectStatus() .isOk(); requestBody = new NewSchemaSubjectDTO() .schemaType(SchemaTypeDTO.PROTOBUF) .subject(subject) .schema(""" syntax = "proto3"; import "MyRecord.proto"; message MyRecordWithRef { int32 id = 1; MyRecord my_ref = 2; } """) .references(List.of(new SchemaReferenceDTO().name("MyRecord.proto").subject(subject + "-ref").version(1))); SchemaSubjectDTO actual = webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromPublisher(Mono.just(requestBody), NewSchemaSubjectDTO.class)) .exchange() .expectStatus() .isOk() .expectBody(SchemaSubjectDTO.class) .returnResult() .getResponseBody(); Assertions.assertNotNull(actual); Assertions.assertEquals(requestBody.getReferences(), actual.getReferences()); } @Test public void shouldReturnBackwardAsGlobalCompatibilityLevelByDefault() { webTestClient .get() .uri("/api/clusters/{clusterName}/schemas/compatibility", LOCAL) .exchange() .expectStatus().isOk() .expectBody(CompatibilityLevelDTO.class) .consumeWith(result -> { CompatibilityLevelDTO responseBody = result.getResponseBody(); Assertions.assertNotNull(responseBody); Assertions.assertEquals(CompatibilityLevelDTO.CompatibilityEnum.BACKWARD, responseBody.getCompatibility()); }); } @Test public void shouldReturnNotEmptyResponseWhenGetAllSchemas() { createNewSubjectAndAssert(subject); webTestClient .get() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .exchange() .expectStatus().isOk() .expectBody(SchemaSubjectsResponseDTO.class) .consumeWith(result -> { SchemaSubjectsResponseDTO responseBody = result.getResponseBody(); log.info("Response of test schemas: {}", responseBody); Assertions.assertNotNull(responseBody); Assertions.assertFalse(responseBody.getSchemas().isEmpty()); SchemaSubjectDTO actualSchemaSubject = responseBody.getSchemas().stream() .filter(schemaSubject -> subject.equals(schemaSubject.getSubject())) .findFirst() .orElseThrow(); Assertions.assertNotNull(actualSchemaSubject.getId()); Assertions.assertNotNull(actualSchemaSubject.getVersion()); Assertions.assertNotNull(actualSchemaSubject.getCompatibilityLevel()); Assertions.assertEquals("\"string\"", actualSchemaSubject.getSchema()); }); } @Test public void shouldOkWhenCreateNewSchemaThenGetAndUpdateItsCompatibilityLevel() { createNewSubjectAndAssert(subject); //Get the created schema and check its items webTestClient .get() .uri("/api/clusters/{clusterName}/schemas/{subject}/latest", LOCAL, subject) .exchange() .expectStatus().isOk() .expectBodyList(SchemaSubjectDTO.class) .consumeWith(listEntityExchangeResult -> { val expectedCompatibility = CompatibilityLevelDTO.CompatibilityEnum.BACKWARD; assertSchemaWhenGetLatest(subject, listEntityExchangeResult, expectedCompatibility); }); // Now let's change compatibility level of this schema to FULL whereas the global // level should be BACKWARD webTestClient.put() .uri("/api/clusters/{clusterName}/schemas/{subject}/compatibility", LOCAL, subject) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue("{\"compatibility\":\"FULL\"}")) .exchange() .expectStatus().isOk(); //Get one more time to check the schema compatibility level is changed to FULL webTestClient .get() .uri("/api/clusters/{clusterName}/schemas/{subject}/latest", LOCAL, subject) .exchange() .expectStatus().isOk() .expectBodyList(SchemaSubjectDTO.class) .consumeWith(listEntityExchangeResult -> { val expectedCompatibility = CompatibilityLevelDTO.CompatibilityEnum.FULL; assertSchemaWhenGetLatest(subject, listEntityExchangeResult, expectedCompatibility); }); } @Test void shouldCreateNewSchemaWhenSubjectIncludesNonAsciiCharacters() { String schema = "{\"subject\":\"test/test\",\"schemaType\":\"JSON\",\"schema\":" + "\"{\\\"type\\\": \\\"string\\\"}\"}"; webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(schema)) .exchange() .expectStatus().isOk(); } private void createNewSubjectAndAssert(String subject) { webTestClient .post() .uri("/api/clusters/{clusterName}/schemas", LOCAL) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue( String.format( "{\"subject\":\"%s\",\"schemaType\":\"AVRO\",\"schema\":" + "\"{\\\"type\\\": \\\"string\\\"}\"}", subject ) )) .exchange() .expectStatus().isOk() .expectBody(SchemaSubjectDTO.class) .consumeWith(this::assertResponseBodyWhenCreateNewSchema); } private void assertSchemaWhenGetLatest( String subject, EntityExchangeResult> listEntityExchangeResult, CompatibilityLevelDTO.CompatibilityEnum expectedCompatibility) { List responseBody = listEntityExchangeResult.getResponseBody(); Assertions.assertNotNull(responseBody); Assertions.assertEquals(1, responseBody.size()); SchemaSubjectDTO actualSchema = responseBody.get(0); Assertions.assertNotNull(actualSchema); Assertions.assertEquals(subject, actualSchema.getSubject()); Assertions.assertEquals("\"string\"", actualSchema.getSchema()); Assertions.assertNotNull(actualSchema.getCompatibilityLevel()); Assertions.assertEquals(SchemaTypeDTO.AVRO, actualSchema.getSchemaType()); Assertions.assertEquals(expectedCompatibility.name(), actualSchema.getCompatibilityLevel()); } private void assertResponseBodyWhenCreateNewSchema( EntityExchangeResult exchangeResult) { SchemaSubjectDTO responseBody = exchangeResult.getResponseBody(); Assertions.assertNotNull(responseBody); Assertions.assertEquals("1", responseBody.getVersion()); Assertions.assertNotNull(responseBody.getSchema()); Assertions.assertNotNull(responseBody.getSubject()); Assertions.assertNotNull(responseBody.getCompatibilityLevel()); Assertions.assertEquals(SchemaTypeDTO.AVRO, responseBody.getSchemaType()); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/config/ClustersPropertiesTest.java ================================================ package com.provectus.kafka.ui.config; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Collections; import org.junit.jupiter.api.Test; class ClustersPropertiesTest { @Test void clusterNamesShouldBeUniq() { ClustersProperties properties = new ClustersProperties(); var c1 = new ClustersProperties.Cluster(); c1.setName("test"); var c2 = new ClustersProperties.Cluster(); c2.setName("test"); //same name Collections.addAll(properties.getClusters(), c1, c2); assertThatThrownBy(properties::validateAndSetDefaults) .hasMessageContaining("Application config isn't valid"); } @Test void clusterNamesShouldSetIfMultipleClustersProvided() { ClustersProperties properties = new ClustersProperties(); var c1 = new ClustersProperties.Cluster(); c1.setName("test1"); var c2 = new ClustersProperties.Cluster(); //name not set Collections.addAll(properties.getClusters(), c1, c2); assertThatThrownBy(properties::validateAndSetDefaults) .hasMessageContaining("Application config isn't valid"); } @Test void ifOnlyOneClusterProvidedNameIsOptionalAndSetToDefault() { ClustersProperties properties = new ClustersProperties(); properties.getClusters().add(new ClustersProperties.Cluster()); properties.validateAndSetDefaults(); assertThat(properties.getClusters()) .element(0) .extracting("name") .isEqualTo("Default"); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/KafkaConnectContainer.java ================================================ package com.provectus.kafka.ui.container; import java.time.Duration; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; public class KafkaConnectContainer extends GenericContainer { private static final int CONNECT_PORT = 8083; public KafkaConnectContainer(String version) { super("confluentinc/cp-kafka-connect:" + version); addExposedPort(CONNECT_PORT); waitStrategy = Wait.forHttp("/") .withStartupTimeout(Duration.ofMinutes(5)); } public KafkaConnectContainer withKafka(KafkaContainer kafka) { String bootstrapServers = kafka.getNetworkAliases().get(0) + ":9092"; return withKafka(kafka.getNetwork(), bootstrapServers); } public KafkaConnectContainer withKafka(Network network, String bootstrapServers) { withNetwork(network); withEnv("CONNECT_BOOTSTRAP_SERVERS", "PLAINTEXT://" + bootstrapServers); withEnv("CONNECT_GROUP_ID", "connect-group"); withEnv("CONNECT_CONFIG_STORAGE_TOPIC", "_connect_configs"); withEnv("CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR", "1"); withEnv("CONNECT_OFFSET_STORAGE_TOPIC", "_connect_offset"); withEnv("CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR", "1"); withEnv("CONNECT_STATUS_STORAGE_TOPIC", "_connect_status"); withEnv("CONNECT_STATUS_STORAGE_REPLICATION_FACTOR", "1"); withEnv("CONNECT_KEY_CONVERTER", "org.apache.kafka.connect.storage.StringConverter"); withEnv("CONNECT_VALUE_CONVERTER", "org.apache.kafka.connect.storage.StringConverter"); withEnv("CONNECT_INTERNAL_KEY_CONVERTER", "org.apache.kafka.connect.json.JsonConverter"); withEnv("CONNECT_INTERNAL_VALUE_CONVERTER", "org.apache.kafka.connect.json.JsonConverter"); withEnv("CONNECT_REST_ADVERTISED_HOST_NAME", "kafka-connect"); withEnv("CONNECT_REST_PORT", String.valueOf(CONNECT_PORT)); withEnv("CONNECT_PLUGIN_PATH", "/usr/share/java,/usr/share/confluent-hub-components," // adding additional paths to find FileStreamSinkConnector + "/usr/local/share/kafka/plugins,/usr/share/filestream-connectors"); return self(); } public String getTarget() { return "http://" + getContainerIpAddress() + ":" + getMappedPort(CONNECT_PORT); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/KsqlDbContainer.java ================================================ package com.provectus.kafka.ui.container; import java.time.Duration; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; public class KsqlDbContainer extends GenericContainer { private static final int PORT = 8088; public KsqlDbContainer(DockerImageName imageName) { super(imageName); addExposedPort(PORT); waitStrategy = Wait .forHttp("/info") .forStatusCode(200) .withStartupTimeout(Duration.ofMinutes(5)); } public KsqlDbContainer withKafka(KafkaContainer kafka) { dependsOn(kafka); String bootstrapServers = kafka.getNetworkAliases().get(0) + ":9092"; return withKafka(kafka.getNetwork(), bootstrapServers); } private KsqlDbContainer withKafka(Network network, String bootstrapServers) { withNetwork(network); withEnv("KSQL_LISTENERS", "http://0.0.0.0:" + PORT); withEnv("KSQL_BOOTSTRAP_SERVERS", bootstrapServers); return self(); } public String url() { return "http://" + getContainerIpAddress() + ":" + getMappedPort(PORT); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/container/SchemaRegistryContainer.java ================================================ package com.provectus.kafka.ui.container; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.Network; public class SchemaRegistryContainer extends GenericContainer { private static final int SCHEMA_PORT = 8081; public SchemaRegistryContainer(String version) { super("confluentinc/cp-schema-registry:" + version); withExposedPorts(8081); } public SchemaRegistryContainer withKafka(KafkaContainer kafka) { String bootstrapServers = kafka.getNetworkAliases().get(0) + ":9092"; return withKafka(kafka.getNetwork(), bootstrapServers); } public SchemaRegistryContainer withKafka(Network network, String bootstrapServers) { withNetwork(network); withEnv("SCHEMA_REGISTRY_HOST_NAME", "schema-registry"); withEnv("SCHEMA_REGISTRY_LISTENERS", "http://0.0.0.0:" + SCHEMA_PORT); withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "PLAINTEXT://" + bootstrapServers); return self(); } public String getUrl() { return "http://" + getContainerIpAddress() + ":" + getMappedPort(SCHEMA_PORT); } public SchemaRegistryClient schemaRegistryClient() { return new CachedSchemaRegistryClient(getUrl(), 1000); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/controller/ApplicationConfigControllerTest.java ================================================ package com.provectus.kafka.ui.controller; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.UploadedFileInfoDTO; import java.io.IOException; import java.nio.file.Path; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.http.HttpEntity; import org.springframework.http.client.MultipartBodyBuilder; import org.springframework.test.web.reactive.server.WebTestClient; import org.springframework.util.MultiValueMap; class ApplicationConfigControllerTest extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @Test public void testUpload() throws IOException { var fileToUpload = new ClassPathResource("/fileForUploadTest.txt", this.getClass()); UploadedFileInfoDTO result = webTestClient .post() .uri("/api/config/relatedfiles") .bodyValue(generateBody(fileToUpload)) .exchange() .expectStatus() .isOk() .expectBody(UploadedFileInfoDTO.class) .returnResult() .getResponseBody(); assertThat(result).isNotNull(); assertThat(result.getLocation()).isNotNull(); assertThat(Path.of(result.getLocation())) .hasSameBinaryContentAs(fileToUpload.getFile().toPath()); } private MultiValueMap> generateBody(ClassPathResource resource) { MultipartBodyBuilder builder = new MultipartBodyBuilder(); builder.part("file", resource); return builder.build(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/MessageFiltersTest.java ================================================ package com.provectus.kafka.ui.emitter; import static com.provectus.kafka.ui.emitter.MessageFilters.containsStringFilter; import static com.provectus.kafka.ui.emitter.MessageFilters.groovyScriptFilter; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.TopicMessageDTO; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.Predicate; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class MessageFiltersTest { @Nested class StringContainsFilter { Predicate filter = containsStringFilter("abC"); @Test void returnsTrueWhenStringContainedInKeyOrContentOrInBoth() { assertTrue( filter.test(msg().key("contains abCd").content("some str")) ); assertTrue( filter.test(msg().key("some str").content("contains abCd")) ); assertTrue( filter.test(msg().key("contains abCd").content("contains abCd")) ); } @Test void returnsFalseOtherwise() { assertFalse( filter.test(msg().key("some str").content("some str")) ); assertFalse( filter.test(msg().key(null).content(null)) ); assertFalse( filter.test(msg().key("aBc").content("AbC")) ); } } @Nested class GroovyScriptFilter { @Test void throwsExceptionOnInvalidGroovySyntax() { assertThrows(ValidationException.class, () -> groovyScriptFilter("this is invalid groovy syntax = 1")); } @Test void canCheckPartition() { var f = groovyScriptFilter("partition == 1"); assertTrue(f.test(msg().partition(1))); assertFalse(f.test(msg().partition(0))); } @Test void canCheckOffset() { var f = groovyScriptFilter("offset == 100"); assertTrue(f.test(msg().offset(100L))); assertFalse(f.test(msg().offset(200L))); } @Test void canCheckHeaders() { var f = groovyScriptFilter("headers.size() == 2 && headers['k1'] == 'v1'"); assertTrue(f.test(msg().headers(Map.of("k1", "v1", "k2", "v2")))); assertFalse(f.test(msg().headers(Map.of("k1", "unexpected", "k2", "v2")))); } @Test void canCheckTimestampMs() { var ts = OffsetDateTime.now(); var f = groovyScriptFilter("timestampMs == " + ts.toInstant().toEpochMilli()); assertTrue(f.test(msg().timestamp(ts))); assertFalse(f.test(msg().timestamp(ts.plus(1L, ChronoUnit.SECONDS)))); } @Test void canCheckValueAsText() { var f = groovyScriptFilter("valueAsText == 'some text'"); assertTrue(f.test(msg().content("some text"))); assertFalse(f.test(msg().content("some other text"))); } @Test void canCheckKeyAsText() { var f = groovyScriptFilter("keyAsText == 'some text'"); assertTrue(f.test(msg().key("some text"))); assertFalse(f.test(msg().key("some other text"))); } @Test void canCheckKeyAsJsonObjectIfItCanBeParsedToJson() { var f = groovyScriptFilter("key.name.first == 'user1'"); assertTrue(f.test(msg().key("{ \"name\" : { \"first\" : \"user1\" } }"))); assertFalse(f.test(msg().key("{ \"name\" : { \"first\" : \"user2\" } }"))); } @Test void keySetToKeyStringIfCantBeParsedToJson() { var f = groovyScriptFilter("key == \"not json\""); assertTrue(f.test(msg().key("not json"))); } @Test void keyAndKeyAsTextSetToNullIfRecordsKeyIsNull() { var f = groovyScriptFilter("key == null"); assertTrue(f.test(msg().key(null))); f = groovyScriptFilter("keyAsText == null"); assertTrue(f.test(msg().key(null))); } @Test void canCheckValueAsJsonObjectIfItCanBeParsedToJson() { var f = groovyScriptFilter("value.name.first == 'user1'"); assertTrue(f.test(msg().content("{ \"name\" : { \"first\" : \"user1\" } }"))); assertFalse(f.test(msg().content("{ \"name\" : { \"first\" : \"user2\" } }"))); } @Test void valueSetToContentStringIfCantBeParsedToJson() { var f = groovyScriptFilter("value == \"not json\""); assertTrue(f.test(msg().content("not json"))); } @Test void valueAndValueAsTextSetToNullIfRecordsContentIsNull() { var f = groovyScriptFilter("value == null"); assertTrue(f.test(msg().content(null))); f = groovyScriptFilter("valueAsText == null"); assertTrue(f.test(msg().content(null))); } @Test void canRunMultiStatementScripts() { var f = groovyScriptFilter("def name = value.name.first \n return name == 'user1' "); assertTrue(f.test(msg().content("{ \"name\" : { \"first\" : \"user1\" } }"))); assertFalse(f.test(msg().content("{ \"name\" : { \"first\" : \"user2\" } }"))); f = groovyScriptFilter("def name = value.name.first; return name == 'user1' "); assertTrue(f.test(msg().content("{ \"name\" : { \"first\" : \"user1\" } }"))); assertFalse(f.test(msg().content("{ \"name\" : { \"first\" : \"user2\" } }"))); f = groovyScriptFilter("def name = value.name.first; name == 'user1' "); assertTrue(f.test(msg().content("{ \"name\" : { \"first\" : \"user1\" } }"))); assertFalse(f.test(msg().content("{ \"name\" : { \"first\" : \"user2\" } }"))); } @Test void filterSpeedIsAtLeast5kPerSec() { var f = groovyScriptFilter("value.name.first == 'user1' && keyAsText.startsWith('a') "); List toFilter = new ArrayList<>(); for (int i = 0; i < 5_000; i++) { String name = i % 2 == 0 ? "user1" : RandomStringUtils.randomAlphabetic(10); String randString = RandomStringUtils.randomAlphabetic(30); String jsonContent = String.format( "{ \"name\" : { \"randomStr\": \"%s\", \"first\" : \"%s\"} }", randString, name); toFilter.add(msg().content(jsonContent).key(randString)); } // first iteration for warmup toFilter.stream().filter(f).count(); long before = System.currentTimeMillis(); long matched = toFilter.stream().filter(f).count(); long took = System.currentTimeMillis() - before; assertThat(took).isLessThan(1000); assertThat(matched).isGreaterThan(0); } } private TopicMessageDTO msg() { return new TopicMessageDTO() .timestamp(OffsetDateTime.now()) .partition(1); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/MessagesProcessingTest.java ================================================ package com.provectus.kafka.ui.emitter; import static org.assertj.core.api.Assertions.assertThat; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.RepeatedTest; class MessagesProcessingTest { @RepeatedTest(5) void testSortingAsc() { var messagesInOrder = List.of( consumerRecord(1, 100L, "1999-01-01T00:00:00+00:00"), consumerRecord(0, 0L, "2000-01-01T00:00:00+00:00"), consumerRecord(1, 200L, "2000-01-05T00:00:00+00:00"), consumerRecord(0, 10L, "2000-01-10T00:00:00+00:00"), consumerRecord(0, 20L, "2000-01-20T00:00:00+00:00"), consumerRecord(1, 300L, "3000-01-01T00:00:00+00:00"), consumerRecord(2, 1000L, "4000-01-01T00:00:00+00:00"), consumerRecord(2, 1001L, "2000-01-01T00:00:00+00:00"), consumerRecord(2, 1003L, "3000-01-01T00:00:00+00:00") ); var shuffled = new ArrayList<>(messagesInOrder); Collections.shuffle(shuffled); var sortedList = MessagesProcessing.sortForSending(shuffled, true); assertThat(sortedList).containsExactlyElementsOf(messagesInOrder); } @RepeatedTest(5) void testSortingDesc() { var messagesInOrder = List.of( consumerRecord(1, 300L, "3000-01-01T00:00:00+00:00"), consumerRecord(2, 1003L, "3000-01-01T00:00:00+00:00"), consumerRecord(0, 20L, "2000-01-20T00:00:00+00:00"), consumerRecord(0, 10L, "2000-01-10T00:00:00+00:00"), consumerRecord(1, 200L, "2000-01-05T00:00:00+00:00"), consumerRecord(0, 0L, "2000-01-01T00:00:00+00:00"), consumerRecord(2, 1001L, "2000-01-01T00:00:00+00:00"), consumerRecord(2, 1000L, "4000-01-01T00:00:00+00:00"), consumerRecord(1, 100L, "1999-01-01T00:00:00+00:00") ); var shuffled = new ArrayList<>(messagesInOrder); Collections.shuffle(shuffled); var sortedList = MessagesProcessing.sortForSending(shuffled, false); assertThat(sortedList).containsExactlyElementsOf(messagesInOrder); } private ConsumerRecord consumerRecord(int partition, long offset, String ts) { return new ConsumerRecord<>( "topic", partition, offset, OffsetDateTime.parse(ts).toInstant().toEpochMilli(), TimestampType.CREATE_TIME, 0, 0, null, null, new RecordHeaders(), Optional.empty() ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/OffsetsInfoTest.java ================================================ package com.provectus.kafka.ui.emitter; import static org.assertj.core.api.Assertions.assertThat; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.kafka.clients.consumer.MockConsumer; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class OffsetsInfoTest { final String topic = "test"; final TopicPartition tp0 = new TopicPartition(topic, 0); //offsets: start 0, end 0 final TopicPartition tp1 = new TopicPartition(topic, 1); //offsets: start 10, end 10 final TopicPartition tp2 = new TopicPartition(topic, 2); //offsets: start 0, end 20 final TopicPartition tp3 = new TopicPartition(topic, 3); //offsets: start 25, end 30 MockConsumer consumer; @BeforeEach void initMockConsumer() { consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); consumer.updatePartitions( topic, Stream.of(tp0, tp1, tp2, tp3) .map(tp -> new PartitionInfo(topic, tp.partition(), null, null, null, null)) .collect(Collectors.toList())); consumer.updateBeginningOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 0L, tp3, 25L)); consumer.updateEndOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 20L, tp3, 30L)); } @Test void fillsInnerFieldsAccordingToTopicState() { var offsets = new OffsetsInfo(consumer, List.of(tp0, tp1, tp2, tp3)); assertThat(offsets.getBeginOffsets()).containsEntry(tp0, 0L).containsEntry(tp1, 10L).containsEntry(tp2, 0L) .containsEntry(tp3, 25L); assertThat(offsets.getEndOffsets()).containsEntry(tp0, 0L).containsEntry(tp1, 10L).containsEntry(tp2, 20L) .containsEntry(tp3, 30L); assertThat(offsets.getEmptyPartitions()).contains(tp0, tp1); assertThat(offsets.getNonEmptyPartitions()).contains(tp2, tp3); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/SeekOperationsTest.java ================================================ package com.provectus.kafka.ui.emitter; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.model.SeekTypeDTO; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.kafka.clients.consumer.MockConsumer; import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class SeekOperationsTest { final String topic = "test"; final TopicPartition tp0 = new TopicPartition(topic, 0); //offsets: start 0, end 0 final TopicPartition tp1 = new TopicPartition(topic, 1); //offsets: start 10, end 10 final TopicPartition tp2 = new TopicPartition(topic, 2); //offsets: start 0, end 20 final TopicPartition tp3 = new TopicPartition(topic, 3); //offsets: start 25, end 30 MockConsumer consumer; @BeforeEach void initMockConsumer() { consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST); consumer.updatePartitions( topic, Stream.of(tp0, tp1, tp2, tp3) .map(tp -> new PartitionInfo(topic, tp.partition(), null, null, null, null)) .collect(Collectors.toList())); consumer.updateBeginningOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 0L, tp3, 25L)); consumer.updateEndOffsets(Map.of(tp0, 0L, tp1, 10L, tp2, 20L, tp3, 30L)); } @Nested class GetOffsetsForSeek { @Test void latest() { var offsets = SeekOperations.getOffsetsForSeek( consumer, new OffsetsInfo(consumer, topic), SeekTypeDTO.LATEST, null ); assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 20L, tp3, 30L)); } @Test void beginning() { var offsets = SeekOperations.getOffsetsForSeek( consumer, new OffsetsInfo(consumer, topic), SeekTypeDTO.BEGINNING, null ); assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 0L, tp3, 25L)); } @Test void offsets() { var offsets = SeekOperations.getOffsetsForSeek( consumer, new OffsetsInfo(consumer, topic), SeekTypeDTO.OFFSET, Map.of(tp1, 10L, tp2, 10L, tp3, 26L) ); assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 10L, tp3, 26L)); } @Test void offsetsWithBoundsFixing() { var offsets = SeekOperations.getOffsetsForSeek( consumer, new OffsetsInfo(consumer, topic), SeekTypeDTO.OFFSET, Map.of(tp1, 10L, tp2, 21L, tp3, 24L) ); assertThat(offsets).containsExactlyInAnyOrderEntriesOf(Map.of(tp2, 20L, tp3, 25L)); } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/emitter/TailingEmitterTest.java ================================================ package com.provectus.kafka.ui.emitter; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.MessageFilterTypeDTO; import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.SeekTypeDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.service.ClustersStorage; import com.provectus.kafka.ui.service.MessagesService; import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; import reactor.core.Disposable; import reactor.core.publisher.Flux; class TailingEmitterTest extends AbstractIntegrationTest { private String topic; private KafkaProducer producer; private Disposable tailingFluxDispose; @BeforeEach void init() { topic = "TopicTailingTest_" + UUID.randomUUID(); createTopic(new NewTopic(topic, 2, (short) 1)); producer = new KafkaProducer<>( Map.of( ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers(), ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class, ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class )); } @AfterEach void tearDown() { deleteTopic(topic); if (tailingFluxDispose != null) { tailingFluxDispose.dispose(); } } @Test void allNewMessagesShouldBeEmitted() throws Exception { var fluxOutput = startTailing(null); List expectedValues = new ArrayList<>(); for (int i = 0; i < 50; i++) { producer.send(new ProducerRecord<>(topic, i + "", i + "")).get(); expectedValues.add(i + ""); } Awaitility.await() .atMost(Duration.ofSeconds(60)) .pollInSameThread() .untilAsserted(() -> assertThat(fluxOutput) .filteredOn(msg -> msg.getType() == TopicMessageEventDTO.TypeEnum.MESSAGE) .extracting(msg -> msg.getMessage().getContent()) .hasSameElementsAs(expectedValues) ); } @Test void allNewMessageThatFitFilterConditionShouldBeEmitted() throws Exception { var fluxOutput = startTailing("good"); List expectedValues = new ArrayList<>(); for (int i = 0; i < 50; i++) { if (i % 2 == 0) { producer.send(new ProducerRecord<>(topic, i + "", i + "_good")).get(); expectedValues.add(i + "_good"); } else { producer.send(new ProducerRecord<>(topic, i + "", i + "_bad")).get(); } } Awaitility.await() .atMost(Duration.ofSeconds(60)) .pollInSameThread() .untilAsserted(() -> assertThat(fluxOutput) .filteredOn(msg -> msg.getType() == TopicMessageEventDTO.TypeEnum.MESSAGE) .extracting(msg -> msg.getMessage().getContent()) .hasSameElementsAs(expectedValues) ); } private Flux createTailingFlux( String topicName, String query) { var cluster = applicationContext.getBean(ClustersStorage.class) .getClusterByName(LOCAL) .get(); return applicationContext.getBean(MessagesService.class) .loadMessages(cluster, topicName, new ConsumerPosition(SeekTypeDTO.LATEST, topic, null), query, MessageFilterTypeDTO.STRING_CONTAINS, 0, SeekDirectionDTO.TAILING, "String", "String"); } private List startTailing(String filterQuery) { List fluxOutput = new CopyOnWriteArrayList<>(); tailingFluxDispose = createTailingFlux(topic, filterQuery) .doOnNext(fluxOutput::add) .subscribe(); // this is needed to be sure that tailing is initialized // and we can start to produce test messages waitUntilTailingInitialized(fluxOutput); return fluxOutput; } private void waitUntilTailingInitialized(List fluxOutput) { Awaitility.await() .pollInSameThread() .pollDelay(Duration.ofMillis(100)) .atMost(Duration.ofSeconds(200)) .until(() -> fluxOutput.stream() .anyMatch(msg -> msg.getType() == TopicMessageEventDTO.TypeEnum.CONSUMING)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/model/PartitionDistributionStatsTest.java ================================================ package com.provectus.kafka.ui.model; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionInfo; import org.assertj.core.data.Percentage; import org.junit.jupiter.api.Test; class PartitionDistributionStatsTest { @Test void skewCalculatedBasedOnPartitionsCounts() { Node n1 = new Node(1, "n1", 9092); Node n2 = new Node(2, "n2", 9092); Node n3 = new Node(3, "n3", 9092); Node n4 = new Node(4, "n4", 9092); var stats = PartitionDistributionStats.create( Statistics.builder() .clusterDescription( new ReactiveAdminClient.ClusterDescription(null, "test", Set.of(n1, n2, n3), null)) .topicDescriptions( Map.of( "t1", new TopicDescription( "t1", false, List.of( new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), new TopicPartitionInfo(1, n2, List.of(n2, n3), List.of(n2, n3)) ) ), "t2", new TopicDescription( "t2", false, List.of( new TopicPartitionInfo(0, n1, List.of(n1, n2), List.of(n1, n2)), new TopicPartitionInfo(1, null, List.of(n2, n1), List.of(n1)) ) ) ) ) .build(), 4 ); assertThat(stats.getPartitionLeaders()) .containsExactlyInAnyOrderEntriesOf(Map.of(n1, 2, n2, 1)); assertThat(stats.getPartitionsCount()) .containsExactlyInAnyOrderEntriesOf(Map.of(n1, 3, n2, 4, n3, 1)); assertThat(stats.getInSyncPartitions()) .containsExactlyInAnyOrderEntriesOf(Map.of(n1, 3, n2, 3, n3, 1)); // Node(partitions): n1(3), n2(4), n3(1), n4(0) // average partitions cnt = (3+4+1) / 3 = 2.666 (counting only nodes with partitions!) assertThat(stats.getAvgPartitionsPerBroker()) .isCloseTo(2.666, Percentage.withPercentage(1)); assertThat(stats.partitionsSkew(n1)) .isCloseTo(BigDecimal.valueOf(12.5), Percentage.withPercentage(1)); assertThat(stats.partitionsSkew(n2)) .isCloseTo(BigDecimal.valueOf(50), Percentage.withPercentage(1)); assertThat(stats.partitionsSkew(n3)) .isCloseTo(BigDecimal.valueOf(-62.5), Percentage.withPercentage(1)); assertThat(stats.partitionsSkew(n4)) .isCloseTo(BigDecimal.valueOf(-100), Percentage.withPercentage(1)); // Node(leaders): n1(2), n2(1), n3(0), n4(0) // average leaders cnt = (2+1) / 2 = 1.5 (counting only nodes with leaders!) assertThat(stats.leadersSkew(n1)) .isCloseTo(BigDecimal.valueOf(33.33), Percentage.withPercentage(1)); assertThat(stats.leadersSkew(n2)) .isCloseTo(BigDecimal.valueOf(-33.33), Percentage.withPercentage(1)); assertThat(stats.leadersSkew(n3)) .isCloseTo(BigDecimal.valueOf(-100), Percentage.withPercentage(1)); assertThat(stats.leadersSkew(n4)) .isCloseTo(BigDecimal.valueOf(-100), Percentage.withPercentage(1)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/producer/KafkaTestProducer.java ================================================ package com.provectus.kafka.ui.producer; import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.StringSerializer; import org.testcontainers.containers.KafkaContainer; public class KafkaTestProducer implements AutoCloseable { private final KafkaProducer producer; private KafkaTestProducer(KafkaProducer producer) { this.producer = producer; } public static KafkaTestProducer forKafka(KafkaContainer kafkaContainer) { return new KafkaTestProducer<>(new KafkaProducer<>(Map.of( ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers(), ProducerConfig.CLIENT_ID_CONFIG, "KafkaTestProducer", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class, ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class ))); } public CompletableFuture send(String topic, ValueT value) { return send(new ProducerRecord<>(topic, value)); } public CompletableFuture send(ProducerRecord record) { CompletableFuture cf = new CompletableFuture<>(); producer.send(record, (m, e) -> { if (e != null) { cf.completeExceptionally(e); } else { cf.complete(m); } }); return cf; } @Override public void close() { producer.close(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/ConsumerRecordDeserializerTest.java ================================================ package com.provectus.kafka.ui.serdes; import static com.provectus.kafka.ui.serde.api.DeserializeResult.Type.STRING; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import java.util.Map; import java.util.function.UnaryOperator; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.Test; class ConsumerRecordDeserializerTest { @Test void dataMaskingAppliedOnDeserializedMessage() { UnaryOperator maskerMock = mock(); Serde.Deserializer deser = (headers, data) -> new DeserializeResult("test", STRING, Map.of()); var recordDeser = new ConsumerRecordDeserializer("test", deser, "test", deser, "test", deser, deser, maskerMock); recordDeser.deserialize(new ConsumerRecord<>("t", 1, 1L, Bytes.wrap("t".getBytes()), Bytes.wrap("t".getBytes()))); verify(maskerMock).apply(any(TopicMessageDTO.class)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/PropertyResolverImplTest.java ================================================ package com.provectus.kafka.ui.serdes; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import java.util.List; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.boot.context.properties.bind.BindException; import org.springframework.mock.env.MockEnvironment; class PropertyResolverImplTest { private static final String TEST_STRING_VALUE = "testStr"; private static final int TEST_INT_VALUE = 123; private static final List TEST_STRING_LIST = List.of("v1", "v2", "v3"); private static final List TEST_INT_LIST = List.of(1, 2, 3); private final MockEnvironment env = new MockEnvironment(); @Data @AllArgsConstructor public static class CustomPropertiesClass { private String f1; private Integer f2; } @Test void returnsEmptyOptionalWhenPropertyNotExist() { var resolver = new PropertyResolverImpl(env); assertThat(resolver.getProperty("nonExistingProp", String.class)).isEmpty(); assertThat(resolver.getListProperty("nonExistingProp", String.class)).isEmpty(); assertThat(resolver.getMapProperty("nonExistingProp", String.class, String.class)).isEmpty(); } @Test void throwsExceptionWhenPropertyCantBeResolverToRequstedClass() { env.setProperty("prop.0.strProp", "testStr"); env.setProperty("prop.0.strLst", "v1,v2,v3"); env.setProperty("prop.0.strMap.k1", "v1"); var resolver = new PropertyResolverImpl(env); assertThatCode(() -> resolver.getProperty("prop.0.strProp", Integer.class)) .isInstanceOf(BindException.class); assertThatCode(() -> resolver.getListProperty("prop.0.strLst", Integer.class)) .isInstanceOf(BindException.class); assertThatCode(() -> resolver.getMapProperty("prop.0.strMap", Integer.class, String.class)) .isInstanceOf(BindException.class); } @Test void resolvedSingleValueProperties() { env.setProperty("prop.0.strProp", "testStr"); env.setProperty("prop.0.intProp", "123"); var resolver = new PropertyResolverImpl(env); assertThat(resolver.getProperty("prop.0.strProp", String.class)) .hasValue("testStr"); assertThat(resolver.getProperty("prop.0.intProp", Integer.class)) .hasValue(123); } @Test void resolvesListProperties() { env.setProperty("prop.0.strLst", "v1,v2,v3"); env.setProperty("prop.0.intLst", "1,2,3"); var resolver = new PropertyResolverImpl(env); assertThat(resolver.getListProperty("prop.0.strLst", String.class)) .hasValue(List.of("v1", "v2", "v3")); assertThat(resolver.getListProperty("prop.0.intLst", Integer.class)) .hasValue(List.of(1, 2, 3)); } @Test void resolvesCustomConfigClassProperties() { env.setProperty("prop.0.custProps.f1", "f1val"); env.setProperty("prop.0.custProps.f2", "1234"); var resolver = new PropertyResolverImpl(env); assertThat(resolver.getProperty("prop.0.custProps", CustomPropertiesClass.class)) .hasValue(new CustomPropertiesClass("f1val", 1234)); } @Test void resolvesMapProperties() { env.setProperty("prop.0.strMap.k1", "v1"); env.setProperty("prop.0.strMap.k2", "v2"); env.setProperty("prop.0.intToLongMap.100", "111"); env.setProperty("prop.0.intToLongMap.200", "222"); var resolver = new PropertyResolverImpl(env); assertThat(resolver.getMapProperty("prop.0.strMap", String.class, String.class)) .hasValue(Map.of("k1", "v1", "k2", "v2")); assertThat(resolver.getMapProperty("prop.0.intToLongMap", Integer.class, Long.class)) .hasValue(Map.of(100, 111L, 200, 222L)); } @Nested class WithPrefix { @Test void resolvedSingleValueProperties() { env.setProperty("prop.0.strProp", "testStr"); env.setProperty("prop.0.intProp", "123"); var resolver = new PropertyResolverImpl(env, "prop.0"); assertThat(resolver.getProperty("strProp", String.class)) .hasValue(TEST_STRING_VALUE); assertThat(resolver.getProperty("intProp", Integer.class)) .hasValue(TEST_INT_VALUE); } @Test void resolvesListProperties() { env.setProperty("prop.0.strLst", "v1,v2,v3"); env.setProperty("prop.0.intLst", "1,2,3"); var resolver = new PropertyResolverImpl(env, "prop.0"); assertThat(resolver.getListProperty("strLst", String.class)) .hasValue(TEST_STRING_LIST); assertThat(resolver.getListProperty("intLst", Integer.class)) .hasValue(TEST_INT_LIST); } @Test void resolvesCustomConfigClassProperties() { env.setProperty("prop.0.custProps.f1", "f1val"); env.setProperty("prop.0.custProps.f2", "1234"); var resolver = new PropertyResolverImpl(env, "prop.0"); assertThat(resolver.getProperty("custProps", CustomPropertiesClass.class)) .hasValue(new CustomPropertiesClass("f1val", 1234)); } @Test void resolvesMapProperties() { env.setProperty("prop.0.strMap.k1", "v1"); env.setProperty("prop.0.strMap.k2", "v2"); env.setProperty("prop.0.intToLongMap.100", "111"); env.setProperty("prop.0.intToLongMap.200", "222"); var resolver = new PropertyResolverImpl(env, "prop.0."); assertThat(resolver.getMapProperty("strMap", String.class, String.class)) .hasValue(Map.of("k1", "v1", "k2", "v2")); assertThat(resolver.getMapProperty("intToLongMap", Integer.class, Long.class)) .hasValue(Map.of(100, 111L, 200, 222L)); } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/SerdesInitializerTest.java ================================================ package com.provectus.kafka.ui.serdes; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serdes.builtin.Int32Serde; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import java.net.URL; import java.net.URLClassLoader; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import org.springframework.core.env.Environment; import org.springframework.mock.env.MockEnvironment; class SerdesInitializerTest { private final Environment env = new MockEnvironment(); private final CustomSerdeLoader customSerdeLoaderMock = mock(CustomSerdeLoader.class); private final SerdesInitializer initializer = new SerdesInitializer( Map.of( "BuiltIn1", BuiltInSerdeWithAutoconfigure.class, "BuiltIn2", BuiltInSerdeMock2NoAutoConfigure.class, Int32Serde.name(), Int32Serde.class, StringSerde.name(), StringSerde.class ), customSerdeLoaderMock ); @Test void pluggedSerdesInitializedByLoader() { ClustersProperties.SerdeConfig customSerdeConfig = new ClustersProperties.SerdeConfig(); customSerdeConfig.setName("MyPluggedSerde"); customSerdeConfig.setFilePath("/custom.jar"); customSerdeConfig.setClassName("org.test.MyPluggedSerde"); customSerdeConfig.setTopicKeysPattern("keys"); customSerdeConfig.setTopicValuesPattern("values"); when(customSerdeLoaderMock.loadAndConfigure(anyString(), anyString(), any(), any(), any())) .thenReturn(new CustomSerdeLoader.CustomSerde(new StringSerde(), new URLClassLoader(new URL[]{}))); var serdes = init(customSerdeConfig); SerdeInstance customSerdeInstance = serdes.serdes.get("MyPluggedSerde"); verifyPatternsMatch(customSerdeConfig, customSerdeInstance); assertThat(customSerdeInstance.classLoader).isNotNull(); verify(customSerdeLoaderMock).loadAndConfigure( eq(customSerdeConfig.getClassName()), eq(customSerdeConfig.getFilePath()), any(), any(), any() ); } @Test void serdeWithBuiltInNameAndNoPropertiesCantBeInitializedIfSerdeNotSupportAutoConfigure() { ClustersProperties.SerdeConfig serdeConfig = new ClustersProperties.SerdeConfig(); serdeConfig.setName("BuiltIn2"); //auto-configuration not supported serdeConfig.setTopicKeysPattern("keys"); serdeConfig.setTopicValuesPattern("vals"); assertThatCode(() -> initializer.init(env, createProperties(serdeConfig), 0)) .isInstanceOf(ValidationException.class); } @Test void serdeWithBuiltInNameAndNoPropertiesIsAutoConfiguredIfPossible() { ClustersProperties.SerdeConfig serdeConfig = new ClustersProperties.SerdeConfig(); serdeConfig.setName("BuiltIn1"); // supports auto-configuration serdeConfig.setTopicKeysPattern("keys"); serdeConfig.setTopicValuesPattern("vals"); var serdes = init(serdeConfig); SerdeInstance autoConfiguredSerde = serdes.serdes.get("BuiltIn1"); verifyAutoConfigured(autoConfiguredSerde); verifyPatternsMatch(serdeConfig, autoConfiguredSerde); } @Test void serdeWithBuiltInNameAndSetPropertiesAreExplicitlyConfigured() { ClustersProperties.SerdeConfig serdeConfig = new ClustersProperties.SerdeConfig(); serdeConfig.setName("BuiltIn1"); serdeConfig.setProperties(Map.of("any", "property")); serdeConfig.setTopicKeysPattern("keys"); serdeConfig.setTopicValuesPattern("vals"); var serdes = init(serdeConfig); SerdeInstance explicitlyConfiguredSerde = serdes.serdes.get("BuiltIn1"); verifyExplicitlyConfigured(explicitlyConfiguredSerde); verifyPatternsMatch(serdeConfig, explicitlyConfiguredSerde); } @Test void serdeWithCustomNameAndBuiltInClassnameAreExplicitlyConfigured() { ClustersProperties.SerdeConfig serdeConfig = new ClustersProperties.SerdeConfig(); serdeConfig.setName("SomeSerde"); serdeConfig.setClassName(BuiltInSerdeWithAutoconfigure.class.getName()); serdeConfig.setTopicKeysPattern("keys"); serdeConfig.setTopicValuesPattern("vals"); var serdes = init(serdeConfig); SerdeInstance explicitlyConfiguredSerde = serdes.serdes.get("SomeSerde"); verifyExplicitlyConfigured(explicitlyConfiguredSerde); verifyPatternsMatch(serdeConfig, explicitlyConfiguredSerde); } private ClusterSerdes init(ClustersProperties.SerdeConfig... serdeConfigs) { return initializer.init(env, createProperties(serdeConfigs), 0); } private ClustersProperties createProperties(ClustersProperties.SerdeConfig... serdeConfigs) { ClustersProperties.Cluster cluster = new ClustersProperties.Cluster(); cluster.setName("test"); cluster.setSerde(List.of(serdeConfigs)); ClustersProperties clustersProperties = new ClustersProperties(); clustersProperties.setClusters(List.of(cluster)); return clustersProperties; } private void verifyExplicitlyConfigured(SerdeInstance serde) { assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).autoConfigureCheckCalled).isFalse(); assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).autoConfigured).isFalse(); assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).explicitlyConfigured).isTrue(); } private void verifyAutoConfigured(SerdeInstance serde) { assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).autoConfigureCheckCalled).isTrue(); assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).autoConfigured).isTrue(); assertThat(((BuiltInSerdeWithAutoconfigure) serde.serde).explicitlyConfigured).isFalse(); } private void verifyPatternsMatch(ClustersProperties.SerdeConfig config, SerdeInstance serde) { assertThat(serde.topicKeyPattern.pattern()).isEqualTo(config.getTopicKeysPattern()); assertThat(serde.topicValuePattern.pattern()).isEqualTo(config.getTopicValuesPattern()); } static class BuiltInSerdeWithAutoconfigure extends StringSerde { boolean explicitlyConfigured = false; boolean autoConfigured = false; boolean autoConfigureCheckCalled = false; @Override public boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { this.autoConfigureCheckCalled = true; return true; } @Override public void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { this.autoConfigured = true; } @Override public void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { this.explicitlyConfigured = true; } } static class BuiltInSerdeMock2NoAutoConfigure extends BuiltInSerdeWithAutoconfigure { @Override public boolean canBeAutoConfigured(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { this.autoConfigureCheckCalled = true; return false; } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/AvroEmbeddedSerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import io.confluent.kafka.schemaregistry.avro.AvroSchemaUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class AvroEmbeddedSerdeTest { private AvroEmbeddedSerde avroEmbeddedSerde; @BeforeEach void init() { avroEmbeddedSerde = new AvroEmbeddedSerde(); avroEmbeddedSerde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void canDeserializeReturnsTrueForAllTargets(Serde.Target target) { assertThat(avroEmbeddedSerde.canDeserialize("anyTopic", target)) .isTrue(); } @ParameterizedTest @EnumSource void canSerializeReturnsFalseForAllTargets(Serde.Target target) { assertThat(avroEmbeddedSerde.canSerialize("anyTopic", target)) .isFalse(); } @Test void deserializerParsesAvroDataWithEmbeddedSchema() throws Exception { Schema schema = new Schema.Parser().parse(""" { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "field1", "type": "string" }, { "name": "field2", "type": "int" } ] } """ ); GenericRecord record = new GenericData.Record(schema); record.put("field1", "this is test msg"); record.put("field2", 100500); String jsonRecord = new String(AvroSchemaUtils.toJson(record)); byte[] serializedRecordBytes = serializeAvroWithEmbeddedSchema(record); var deserializer = avroEmbeddedSerde.deserializer("anyTopic", Serde.Target.KEY); DeserializeResult result = deserializer.deserialize(null, serializedRecordBytes); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); assertThat(result.getAdditionalProperties()).isEmpty(); assertJsonEquals(jsonRecord, result.getResult()); } private void assertJsonEquals(String expected, String actual) throws IOException { var mapper = new JsonMapper(); assertThat(mapper.readTree(actual)).isEqualTo(mapper.readTree(expected)); } private byte[] serializeAvroWithEmbeddedSchema(GenericRecord record) throws IOException { try (DataFileWriter writer = new DataFileWriter<>(new GenericDatumWriter<>()); ByteArrayOutputStream baos = new ByteArrayOutputStream()) { writer.create(record.getSchema(), baos); writer.append(record); writer.flush(); return baos.toByteArray(); } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/Base64SerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import java.util.Base64; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class Base64SerdeTest { private static final byte[] TEST_BYTES = "some bytes go here".getBytes(); private static final String TEST_BYTES_BASE64_ENCODED = Base64.getEncoder().encodeToString(TEST_BYTES); private Serde base64Serde; @BeforeEach void init() { base64Serde = new Base64Serde(); base64Serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializesInputAsBase64String(Serde.Target type) { var serializer = base64Serde.serializer("anyTopic", type); byte[] bytes = serializer.serialize(TEST_BYTES_BASE64_ENCODED); assertThat(bytes).isEqualTo(TEST_BYTES); } @ParameterizedTest @EnumSource void deserializesDataAsBase64Bytes(Serde.Target type) { var deserializer = base64Serde.deserializer("anyTopic", type); var result = deserializer.deserialize(new RecordHeadersImpl(), TEST_BYTES); assertThat(result.getResult()).isEqualTo(TEST_BYTES_BASE64_ENCODED); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.STRING); assertThat(result.getAdditionalProperties()).isEmpty(); } @ParameterizedTest @EnumSource void getSchemaReturnsEmpty(Serde.Target type) { assertThat(base64Serde.getSchema("anyTopic", type)).isEmpty(); } @ParameterizedTest @EnumSource void canDeserializeReturnsTrueForAllInputs(Serde.Target type) { assertThat(base64Serde.canDeserialize("anyTopic", type)).isTrue(); } @ParameterizedTest @EnumSource void canSerializeReturnsTrueForAllInput(Serde.Target type) { assertThat(base64Serde.canSerialize("anyTopic", type)).isTrue(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/ConsumerOffsetsSerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static com.provectus.kafka.ui.serdes.builtin.ConsumerOffsetsSerde.TOPIC; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.producer.KafkaTestProducer; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.UUID; import lombok.SneakyThrows; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.BytesDeserializer; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; import reactor.util.function.Tuple2; import reactor.util.function.Tuples; class ConsumerOffsetsSerdeTest extends AbstractIntegrationTest { private static final int MSGS_TO_GENERATE = 10; private static String consumerGroupName; private static String committedTopic; @BeforeAll static void createTopicAndCommitItsOffset() { committedTopic = ConsumerOffsetsSerdeTest.class.getSimpleName() + "-" + UUID.randomUUID(); consumerGroupName = committedTopic + "-group"; createTopic(new NewTopic(committedTopic, 1, (short) 1)); try (var producer = KafkaTestProducer.forKafka(kafka)) { for (int i = 0; i < MSGS_TO_GENERATE; i++) { producer.send(committedTopic, "i=" + i); } } try (var consumer = createConsumer(consumerGroupName)) { consumer.subscribe(List.of(committedTopic)); int polled = 0; while (polled < MSGS_TO_GENERATE) { polled += consumer.poll(Duration.ofMillis(100)).count(); } consumer.commitSync(); } } @AfterAll static void cleanUp() { deleteTopic(committedTopic); } @Test void canOnlyDeserializeConsumerOffsetsTopic() { var serde = new ConsumerOffsetsSerde(); assertThat(serde.canDeserialize(ConsumerOffsetsSerde.TOPIC, Serde.Target.KEY)).isTrue(); assertThat(serde.canDeserialize(ConsumerOffsetsSerde.TOPIC, Serde.Target.VALUE)).isTrue(); assertThat(serde.canDeserialize("anyOtherTopic", Serde.Target.KEY)).isFalse(); assertThat(serde.canDeserialize("anyOtherTopic", Serde.Target.VALUE)).isFalse(); } @Test void deserializesMessagesMadeByConsumerActivity() { var serde = new ConsumerOffsetsSerde(); var keyDeserializer = serde.deserializer(TOPIC, Serde.Target.KEY); var valueDeserializer = serde.deserializer(TOPIC, Serde.Target.VALUE); try (var consumer = createConsumer(consumerGroupName + "-check")) { consumer.subscribe(List.of(ConsumerOffsetsSerde.TOPIC)); List> polled = new ArrayList<>(); Awaitility.await() .pollInSameThread() .atMost(Duration.ofMinutes(1)) .untilAsserted(() -> { for (var rec : consumer.poll(Duration.ofMillis(200))) { DeserializeResult key = rec.key() != null ? keyDeserializer.deserialize(null, rec.key().get()) : null; DeserializeResult val = rec.value() != null ? valueDeserializer.deserialize(null, rec.value().get()) : null; if (key != null && val != null) { polled.add(Tuples.of(key, val)); } } assertThat(polled).anyMatch(t -> isCommitMessage(t.getT1(), t.getT2())); assertThat(polled).anyMatch(t -> isGroupMetadataMessage(t.getT1(), t.getT2())); }); } } // Sample commit record: // // key: { // "group": "test_Members_3", // "topic": "test", // "partition": 0 // } // // value: // { // "offset": 2, // "leader_epoch": 0, // "metadata": "", // "commit_timestamp": 1683112980588 // } private boolean isCommitMessage(DeserializeResult key, DeserializeResult value) { var keyJson = toMapFromJsom(key); boolean keyIsOk = consumerGroupName.equals(keyJson.get("group")) && committedTopic.equals(keyJson.get("topic")) && ((Integer) 0).equals(keyJson.get("partition")); var valueJson = toMapFromJsom(value); boolean valueIsOk = valueJson.containsKey("offset") && valueJson.get("offset").equals(MSGS_TO_GENERATE) && valueJson.containsKey("commit_timestamp"); return keyIsOk && valueIsOk; } // Sample group metadata record: // // key: { // "group": "test_Members_3" // } // // value: // { // "protocol_type": "consumer", // "generation": 1, // "protocol": "range", // "leader": "consumer-test_Members_3-1-5a37876e-e42f-420e-9c7d-6902889bd5dd", // "current_state_timestamp": 1683112974561, // "members": [ // { // "member_id": "consumer-test_Members_3-1-5a37876e-e42f-420e-9c7d-6902889bd5dd", // "group_instance_id": null, // "client_id": "consumer-test_Members_3-1", // "client_host": "/192.168.16.1", // "rebalance_timeout": 300000, // "session_timeout": 45000, // "subscription": "AAEAAAABAAR0ZXN0/////wAAAAA=", // "assignment": "AAEAAAABAAR0ZXN0AAAAAQAAAAD/////" // } // ] // } private boolean isGroupMetadataMessage(DeserializeResult key, DeserializeResult value) { var keyJson = toMapFromJsom(key); boolean keyIsOk = consumerGroupName.equals(keyJson.get("group")) && keyJson.size() == 1; var valueJson = toMapFromJsom(value); boolean valueIsOk = valueJson.keySet() .containsAll(Set.of("protocol_type", "generation", "leader", "members")); return keyIsOk && valueIsOk; } @SneakyThrows private Map toMapFromJsom(DeserializeResult result) { return new JsonMapper().readValue(result.getResult(), Map.class); } private static KafkaConsumer createConsumer(String groupId) { Properties props = new Properties(); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); props.put(ConsumerConfig.CLIENT_ID_CONFIG, groupId); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); return new KafkaConsumer<>(props); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/HexSerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; public class HexSerdeTest { private static final byte[] TEST_BYTES = "hello world".getBytes(); private static final String TEST_BYTES_HEX_ENCODED = "68 65 6C 6C 6F 20 77 6F 72 6C 64"; private HexSerde hexSerde; @BeforeEach void init() { hexSerde = new HexSerde(); hexSerde.autoConfigure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); } @ParameterizedTest @CsvSource({ "68656C6C6F20776F726C64", // uppercase "68656c6c6f20776f726c64", // lowercase "68:65:6c:6c:6f:20:77:6f:72:6c:64", // ':' delim "68 65 6C 6C 6F 20 77 6F 72 6C 64", // space delim, UC "68 65 6c 6c 6f 20 77 6f 72 6c 64", // space delim, LC "#68 #65 #6C #6C #6F #20 #77 #6F #72 #6C #64" // '#' prefix, space delim }) void serializesInputAsHexString(String hexString) { for (Serde.Target type : Serde.Target.values()) { var serializer = hexSerde.serializer("anyTopic", type); byte[] bytes = serializer.serialize(hexString); assertThat(bytes).isEqualTo(TEST_BYTES); } } @ParameterizedTest @EnumSource void serializesEmptyStringAsEmptyBytesArray(Serde.Target type) { var serializer = hexSerde.serializer("anyTopic", type); byte[] bytes = serializer.serialize(""); assertThat(bytes).isEqualTo(new byte[] {}); } @ParameterizedTest @EnumSource void deserializesDataAsHexBytes(Serde.Target type) { var deserializer = hexSerde.deserializer("anyTopic", type); var result = deserializer.deserialize(new RecordHeadersImpl(), TEST_BYTES); assertThat(result.getResult()).isEqualTo(TEST_BYTES_HEX_ENCODED); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.STRING); assertThat(result.getAdditionalProperties()).isEmpty(); } @ParameterizedTest @EnumSource void getSchemaReturnsEmpty(Serde.Target type) { assertThat(hexSerde.getSchema("anyTopic", type)).isEmpty(); } @ParameterizedTest @EnumSource void canDeserializeReturnsTrueForAllInputs(Serde.Target type) { assertThat(hexSerde.canDeserialize("anyTopic", type)).isTrue(); } @ParameterizedTest @EnumSource void canSerializeReturnsTrueForAllInput(Serde.Target type) { assertThat(hexSerde.canSerialize("anyTopic", type)).isTrue(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/Int32SerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.primitives.Ints; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import org.apache.kafka.common.header.internals.RecordHeaders; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class Int32SerdeTest { private Int32Serde serde; @BeforeEach void init() { serde = new Int32Serde(); serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializeUses4BytesIntRepresentation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); byte[] bytes = serializer.serialize("1234"); assertThat(bytes).isEqualTo(Ints.toByteArray(1234)); } @ParameterizedTest @EnumSource void deserializeUses4BytesIntRepresentation(Serde.Target type) { var deserializer = serde.deserializer("anyTopic", type); var result = deserializer.deserialize(new RecordHeadersImpl(), Ints.toByteArray(1234)); assertThat(result.getResult()).isEqualTo("1234"); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/Int64SerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.primitives.Longs; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import org.apache.kafka.common.header.internals.RecordHeaders; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class Int64SerdeTest { private Int64Serde serde; @BeforeEach void init() { serde = new Int64Serde(); serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializeUses8BytesLongRepresentation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); byte[] bytes = serializer.serialize("1234"); assertThat(bytes).isEqualTo(Longs.toByteArray(1234)); } @ParameterizedTest @EnumSource void deserializeUses8BytesLongRepresentation(Serde.Target type) { var deserializer = serde.deserializer("anyTopic", type); var result = deserializer.deserialize(new RecordHeadersImpl(), Longs.toByteArray(1234)); assertThat(result.getResult()).isEqualTo("1234"); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/ProtobufFileSerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.json.JsonMapper; import com.google.protobuf.Descriptors; import com.google.protobuf.util.JsonFormat; import com.provectus.kafka.ui.serde.api.PropertyResolver; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.builtin.ProtobufFileSerde.Configuration; import com.squareup.wire.schema.ProtoFile; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.springframework.util.ResourceUtils; class ProtobufFileSerdeTest { private static final String samplePersonMsgJson = "{ \"name\": \"My Name\",\"id\": 101, \"email\": \"user1@example.com\", \"phones\":[] }"; private static final String sampleBookMsgJson = "{\"version\": 1, \"people\": [" + "{ \"name\": \"My Name\",\"id\": 102, \"email\": \"addrBook@example.com\", \"phones\":[]}]}"; private static final String sampleLangDescriptionMsgJson = "{ \"lang\": \"EN\", " + "\"descr\": \"Some description here\" }"; // Sample message of type `test.Person` private byte[] personMessageBytes; // Sample message of type `test.AddressBook` private byte[] addressBookMessageBytes; private byte[] langDescriptionMessageBytes; private Descriptors.Descriptor personDescriptor; private Descriptors.Descriptor addressBookDescriptor; private Descriptors.Descriptor langDescriptionDescriptor; private Map descriptorPaths; @BeforeEach void setUp() throws Exception { Map files = ProtobufFileSerde.Configuration.loadSchemas( Optional.empty(), Optional.of(protoFilesDir()) ); Path addressBookSchemaPath = ResourceUtils.getFile("classpath:protobuf-serde/address-book.proto").toPath(); var addressBookSchema = files.get(addressBookSchemaPath); var builder = addressBookSchema.newMessageBuilder("test.Person"); JsonFormat.parser().merge(samplePersonMsgJson, builder); personMessageBytes = builder.build().toByteArray(); builder = addressBookSchema.newMessageBuilder("test.AddressBook"); JsonFormat.parser().merge(sampleBookMsgJson, builder); addressBookMessageBytes = builder.build().toByteArray(); personDescriptor = addressBookSchema.toDescriptor("test.Person"); addressBookDescriptor = addressBookSchema.toDescriptor("test.AddressBook"); Path languageDescriptionPath = ResourceUtils.getFile("classpath:protobuf-serde/lang-description.proto").toPath(); var languageDescriptionSchema = files.get(languageDescriptionPath); builder = languageDescriptionSchema.newMessageBuilder("test.LanguageDescription"); JsonFormat.parser().merge(sampleLangDescriptionMsgJson, builder); langDescriptionMessageBytes = builder.build().toByteArray(); langDescriptionDescriptor = languageDescriptionSchema.toDescriptor("test.LanguageDescription"); descriptorPaths = Map.of( personDescriptor, addressBookSchemaPath, addressBookDescriptor, addressBookSchemaPath ); } @Test void loadsAllProtoFiledFromTargetDirectory() throws Exception { var protoDir = ResourceUtils.getFile("classpath:protobuf-serde/").getPath(); List files = new ProtobufFileSerde.ProtoSchemaLoader(protoDir).load(); assertThat(files).hasSize(4); assertThat(files) .map(f -> f.getLocation().getPath()) .containsExactlyInAnyOrder( "language/language.proto", "sensor.proto", "address-book.proto", "lang-description.proto" ); } @SneakyThrows private String protoFilesDir() { return ResourceUtils.getFile("classpath:protobuf-serde/").getPath(); } @Nested class ConfigurationTests { @Test void canBeAutoConfiguredReturnsNoProtoPropertiesProvided() { PropertyResolver resolver = mock(PropertyResolver.class); assertThat(Configuration.canBeAutoConfigured(resolver)) .isFalse(); } @Test void canBeAutoConfiguredReturnsTrueIfProtoFilesHasBeenProvided() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getListProperty("protobufFiles", String.class)) .thenReturn(Optional.of(List.of("file.proto"))); assertThat(Configuration.canBeAutoConfigured(resolver)) .isTrue(); } @Test void canBeAutoConfiguredReturnsTrueIfProtoFilesDirProvided() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of("/filesDir")); assertThat(Configuration.canBeAutoConfigured(resolver)) .isTrue(); } @Test void unknownSchemaAsDefaultThrowsException() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of(protoFilesDir())); when(resolver.getProperty("protobufMessageName", String.class)) .thenReturn(Optional.of("test.NotExistent")); assertThatThrownBy(() -> Configuration.create(resolver)) .isInstanceOf(NullPointerException.class) .hasMessage("The given message type not found in protobuf definition: test.NotExistent"); } @Test void unknownSchemaAsDefaultForKeyThrowsException() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of(protoFilesDir())); when(resolver.getProperty("protobufMessageNameForKey", String.class)) .thenReturn(Optional.of("test.NotExistent")); assertThatThrownBy(() -> Configuration.create(resolver)) .isInstanceOf(NullPointerException.class) .hasMessage("The given message type not found in protobuf definition: test.NotExistent"); } @Test void unknownSchemaAsTopicSchemaThrowsException() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of(protoFilesDir())); when(resolver.getMapProperty("protobufMessageNameByTopic", String.class, String.class)) .thenReturn(Optional.of(Map.of("persons", "test.NotExistent"))); assertThatThrownBy(() -> Configuration.create(resolver)) .isInstanceOf(NullPointerException.class) .hasMessage("The given message type not found in protobuf definition: test.NotExistent"); } @Test void unknownSchemaAsTopicSchemaForKeyThrowsException() { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of(protoFilesDir())); when(resolver.getMapProperty("protobufMessageNameForKeyByTopic", String.class, String.class)) .thenReturn(Optional.of(Map.of("persons", "test.NotExistent"))); assertThatThrownBy(() -> Configuration.create(resolver)) .isInstanceOf(NullPointerException.class) .hasMessage("The given message type not found in protobuf definition: test.NotExistent"); } @Test void createConfigureFillsDescriptorMappingsWhenProtoFilesListProvided() throws Exception { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getListProperty("protobufFiles", String.class)) .thenReturn(Optional.of( List.of( ResourceUtils.getFile("classpath:protobuf-serde/sensor.proto").getPath(), ResourceUtils.getFile("classpath:protobuf-serde/address-book.proto").getPath()))); when(resolver.getProperty("protobufMessageName", String.class)) .thenReturn(Optional.of("test.Sensor")); when(resolver.getProperty("protobufMessageNameForKey", String.class)) .thenReturn(Optional.of("test.AddressBook")); when(resolver.getMapProperty("protobufMessageNameByTopic", String.class, String.class)) .thenReturn(Optional.of( Map.of( "topic1", "test.Sensor", "topic2", "test.AddressBook"))); when(resolver.getMapProperty("protobufMessageNameForKeyByTopic", String.class, String.class)) .thenReturn(Optional.of( Map.of( "topic1", "test.Person", "topic2", "test.AnotherPerson"))); var configuration = Configuration.create(resolver); assertThat(configuration.defaultMessageDescriptor()) .matches(d -> d.getFullName().equals("test.Sensor")); assertThat(configuration.defaultKeyMessageDescriptor()) .matches(d -> d.getFullName().equals("test.AddressBook")); assertThat(configuration.messageDescriptorMap()) .containsOnlyKeys("topic1", "topic2") .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.Sensor")) .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.AddressBook")); assertThat(configuration.keyMessageDescriptorMap()) .containsOnlyKeys("topic1", "topic2") .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.Person")) .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.AnotherPerson")); } @Test void createConfigureFillsDescriptorMappingsWhenProtoFileDirProvided() throws Exception { PropertyResolver resolver = mock(PropertyResolver.class); when(resolver.getProperty("protobufFilesDir", String.class)) .thenReturn(Optional.of(protoFilesDir())); when(resolver.getProperty("protobufMessageName", String.class)) .thenReturn(Optional.of("test.Sensor")); when(resolver.getProperty("protobufMessageNameForKey", String.class)) .thenReturn(Optional.of("test.AddressBook")); when(resolver.getMapProperty("protobufMessageNameByTopic", String.class, String.class)) .thenReturn(Optional.of( Map.of( "topic1", "test.Sensor", "topic2", "test.LanguageDescription"))); when(resolver.getMapProperty("protobufMessageNameForKeyByTopic", String.class, String.class)) .thenReturn(Optional.of( Map.of( "topic1", "test.Person", "topic2", "test.AnotherPerson"))); var configuration = Configuration.create(resolver); assertThat(configuration.defaultMessageDescriptor()) .matches(d -> d.getFullName().equals("test.Sensor")); assertThat(configuration.defaultKeyMessageDescriptor()) .matches(d -> d.getFullName().equals("test.AddressBook")); assertThat(configuration.messageDescriptorMap()) .containsOnlyKeys("topic1", "topic2") .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.Sensor")) .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.LanguageDescription")); assertThat(configuration.keyMessageDescriptorMap()) .containsOnlyKeys("topic1", "topic2") .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.Person")) .anySatisfy((topic, descr) -> assertThat(descr.getFullName()).isEqualTo("test.AnotherPerson")); } } @Test void deserializeUsesTopicsMappingToFindMsgDescriptor() { var messageNameMap = Map.of( "persons", personDescriptor, "books", addressBookDescriptor, "langs", langDescriptionDescriptor ); var keyMessageNameMap = Map.of( "books", addressBookDescriptor); var serde = new ProtobufFileSerde(); serde.configure( new Configuration( null, null, descriptorPaths, messageNameMap, keyMessageNameMap ) ); var deserializedPerson = serde.deserializer("persons", Serde.Target.VALUE) .deserialize(null, personMessageBytes); assertJsonEquals(samplePersonMsgJson, deserializedPerson.getResult()); var deserializedBook = serde.deserializer("books", Serde.Target.KEY) .deserialize(null, addressBookMessageBytes); assertJsonEquals(sampleBookMsgJson, deserializedBook.getResult()); var deserializedSensor = serde.deserializer("langs", Serde.Target.VALUE) .deserialize(null, langDescriptionMessageBytes); assertJsonEquals(sampleLangDescriptionMsgJson, deserializedSensor.getResult()); } @Test void deserializeUsesDefaultDescriptorIfTopicMappingNotFound() { var serde = new ProtobufFileSerde(); serde.configure( new Configuration( personDescriptor, addressBookDescriptor, descriptorPaths, Map.of(), Map.of() ) ); var deserializedPerson = serde.deserializer("persons", Serde.Target.VALUE) .deserialize(null, personMessageBytes); assertJsonEquals(samplePersonMsgJson, deserializedPerson.getResult()); var deserializedBook = serde.deserializer("books", Serde.Target.KEY) .deserialize(null, addressBookMessageBytes); assertJsonEquals(sampleBookMsgJson, deserializedBook.getResult()); } @Test void serializeUsesTopicsMappingToFindMsgDescriptor() { var messageNameMap = Map.of( "persons", personDescriptor, "books", addressBookDescriptor, "langs", langDescriptionDescriptor ); var keyMessageNameMap = Map.of( "books", addressBookDescriptor); var serde = new ProtobufFileSerde(); serde.configure( new Configuration( null, null, descriptorPaths, messageNameMap, keyMessageNameMap ) ); var personBytes = serde.serializer("langs", Serde.Target.VALUE) .serialize(sampleLangDescriptionMsgJson); assertThat(personBytes).isEqualTo(langDescriptionMessageBytes); var booksBytes = serde.serializer("books", Serde.Target.KEY) .serialize(sampleBookMsgJson); assertThat(booksBytes).isEqualTo(addressBookMessageBytes); } @Test void serializeUsesDefaultDescriptorIfTopicMappingNotFound() { var serde = new ProtobufFileSerde(); serde.configure( new Configuration( personDescriptor, addressBookDescriptor, descriptorPaths, Map.of(), Map.of() ) ); var personBytes = serde.serializer("persons", Serde.Target.VALUE) .serialize(samplePersonMsgJson); assertThat(personBytes).isEqualTo(personMessageBytes); var booksBytes = serde.serializer("books", Serde.Target.KEY) .serialize(sampleBookMsgJson); assertThat(booksBytes).isEqualTo(addressBookMessageBytes); } @SneakyThrows private void assertJsonEquals(String expectedJson, String actualJson) { var mapper = new JsonMapper(); assertThat(mapper.readTree(actualJson)).isEqualTo(mapper.readTree(expectedJson)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/ProtobufRawSerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.serde.api.Serde; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class ProtobufRawSerdeTest { private static final String DUMMY_TOPIC = "dummy-topic"; private ProtobufRawSerde serde; @BeforeEach void init() { serde = new ProtobufRawSerde(); } @SneakyThrows ProtobufSchema getSampleSchema() { return new ProtobufSchema( """ syntax = "proto3"; message Message1 { int32 my_field = 1; } """ ); } @SneakyThrows private byte[] getProtobufMessage() { DynamicMessage.Builder builder = DynamicMessage.newBuilder(getSampleSchema().toDescriptor("Message1")); builder.setField(builder.getDescriptorForType().findFieldByName("my_field"), 5); return builder.build().toByteArray(); } @Test void deserializeSimpleMessage() { var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE) .deserialize(null, getProtobufMessage()); assertThat(deserialized.getResult()).isEqualTo("1: 5\n"); } @Test void deserializeEmptyMessage() { var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE) .deserialize(null, new byte[0]); assertThat(deserialized.getResult()).isEqualTo(""); } @Test void deserializeInvalidMessage() { var deserializer = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE); assertThatThrownBy(() -> deserializer.deserialize(null, new byte[] { 1, 2, 3 })) .isInstanceOf(ValidationException.class) .hasMessageContaining("Protocol message contained an invalid tag"); } @Test void deserializeNullMessage() { var deserializer = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE); assertThatThrownBy(() -> deserializer.deserialize(null, null)) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot read the array length"); } ProtobufSchema getSampleNestedSchema() { return new ProtobufSchema( """ syntax = "proto3"; message Message2 { int32 my_nested_field = 1; } message Message1 { int32 my_field = 1; Message2 my_nested_message = 2; } """ ); } @SneakyThrows private byte[] getComplexProtobufMessage() { DynamicMessage.Builder builder = DynamicMessage.newBuilder(getSampleNestedSchema().toDescriptor("Message1")); builder.setField(builder.getDescriptorForType().findFieldByName("my_field"), 5); DynamicMessage.Builder nestedBuilder = DynamicMessage.newBuilder(getSampleNestedSchema().toDescriptor("Message2")); nestedBuilder.setField(nestedBuilder.getDescriptorForType().findFieldByName("my_nested_field"), 10); builder.setField(builder.getDescriptorForType().findFieldByName("my_nested_message"), nestedBuilder.build()); return builder.build().toByteArray(); } @Test void deserializeNestedMessage() { var deserialized = serde.deserializer(DUMMY_TOPIC, Serde.Target.VALUE) .deserialize(null, getComplexProtobufMessage()); assertThat(deserialized.getResult()).isEqualTo("1: 5\n2: {\n 1: 10\n}\n"); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/UInt32SerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.primitives.Ints; import com.google.common.primitives.UnsignedInteger; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import org.apache.kafka.common.header.internals.RecordHeaders; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class UInt32SerdeTest { private UInt32Serde serde; @BeforeEach void init() { serde = new UInt32Serde(); serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializeUses4BytesUInt32Representation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); String uint32String = UnsignedInteger.MAX_VALUE.toString(); byte[] bytes = serializer.serialize(uint32String); assertThat(bytes).isEqualTo(Ints.toByteArray(UnsignedInteger.MAX_VALUE.intValue())); } @ParameterizedTest @EnumSource void serializeThrowsNfeIfNegativeValuePassed(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); String negativeIntString = "-100"; assertThatThrownBy(() -> serializer.serialize(negativeIntString)) .isInstanceOf(NumberFormatException.class); } @ParameterizedTest @EnumSource void deserializeUses4BytesUInt32Representation(Serde.Target type) { var deserializer = serde.deserializer("anyTopic", type); byte[] uint32Bytes = Ints.toByteArray(UnsignedInteger.MAX_VALUE.intValue()); var result = deserializer.deserialize(new RecordHeadersImpl(), uint32Bytes); assertThat(result.getResult()).isEqualTo(UnsignedInteger.MAX_VALUE.toString()); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/UInt64SerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.primitives.Longs; import com.google.common.primitives.UnsignedLong; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; class UInt64SerdeTest { private UInt64Serde serde; @BeforeEach void init() { serde = new UInt64Serde(); serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializeUses8BytesUInt64Representation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); String uint64String = UnsignedLong.MAX_VALUE.toString(); byte[] bytes = serializer.serialize(uint64String); assertThat(bytes).isEqualTo(Longs.toByteArray(UnsignedLong.MAX_VALUE.longValue())); } @ParameterizedTest @EnumSource void serializeThrowsNfeIfNegativeValuePassed(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); String negativeIntString = "-100"; assertThatThrownBy(() -> serializer.serialize(negativeIntString)) .isInstanceOf(NumberFormatException.class); } @ParameterizedTest @EnumSource void deserializeUses8BytesUIn64tRepresentation(Serde.Target type) { var deserializer = serde.deserializer("anyTopic", type); byte[] uint64Bytes = Longs.toByteArray(UnsignedLong.MAX_VALUE.longValue()); var result = deserializer.deserialize(new RecordHeadersImpl(), uint64Bytes); assertThat(result.getResult()).isEqualTo(UnsignedLong.MAX_VALUE.toString()); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/UuidBinarySerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.RecordHeadersImpl; import java.nio.ByteBuffer; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.springframework.mock.env.MockEnvironment; class UuidBinarySerdeTest { @Nested class MsbFirst { private UuidBinarySerde serde; @BeforeEach void init() { serde = new UuidBinarySerde(); serde.configure( PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializerUses16bytesUuidBinaryRepresentation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); var uuid = UUID.randomUUID(); byte[] bytes = serializer.serialize(uuid.toString()); var bb = ByteBuffer.wrap(bytes); assertThat(bb.getLong()).isEqualTo(uuid.getMostSignificantBits()); assertThat(bb.getLong()).isEqualTo(uuid.getLeastSignificantBits()); } @ParameterizedTest @EnumSource void deserializerUses16bytesUuidBinaryRepresentation(Serde.Target type) { var uuid = UUID.randomUUID(); var bb = ByteBuffer.allocate(16); bb.putLong(uuid.getMostSignificantBits()); bb.putLong(uuid.getLeastSignificantBits()); var result = serde.deserializer("anyTopic", type).deserialize(new RecordHeadersImpl(), bb.array()); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.STRING); assertThat(result.getAdditionalProperties()).isEmpty(); assertThat(result.getResult()).isEqualTo(uuid.toString()); } } @Nested class MsbLast { private UuidBinarySerde serde; @BeforeEach void init() { serde = new UuidBinarySerde(); serde.configure( new PropertyResolverImpl(new MockEnvironment().withProperty("mostSignificantBitsFirst", "false")), PropertyResolverImpl.empty(), PropertyResolverImpl.empty() ); } @ParameterizedTest @EnumSource void serializerUses16bytesUuidBinaryRepresentation(Serde.Target type) { var serializer = serde.serializer("anyTopic", type); var uuid = UUID.randomUUID(); byte[] bytes = serializer.serialize(uuid.toString()); var bb = ByteBuffer.wrap(bytes); assertThat(bb.getLong()).isEqualTo(uuid.getLeastSignificantBits()); assertThat(bb.getLong()).isEqualTo(uuid.getMostSignificantBits()); } @ParameterizedTest @EnumSource void deserializerUses16bytesUuidBinaryRepresentation(Serde.Target type) { var uuid = UUID.randomUUID(); var bb = ByteBuffer.allocate(16); bb.putLong(uuid.getLeastSignificantBits()); bb.putLong(uuid.getMostSignificantBits()); var result = serde.deserializer("anyTopic", type).deserialize(new RecordHeadersImpl(), bb.array()); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.STRING); assertThat(result.getAdditionalProperties()).isEmpty(); assertThat(result.getResult()).isEqualTo(uuid.toString()); } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/serdes/builtin/sr/SchemaRegistrySerdeTest.java ================================================ package com.provectus.kafka.ui.serdes.builtin.sr; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.serde.api.DeserializeResult; import com.provectus.kafka.ui.serde.api.SchemaDescription; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.util.jsonschema.JsonAvroConversion; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import lombok.SneakyThrows; import net.bytebuddy.utility.RandomString; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; class SchemaRegistrySerdeTest { private final MockSchemaRegistryClient registryClient = new MockSchemaRegistryClient(); private SchemaRegistrySerde serde; @BeforeEach void init() { serde = new SchemaRegistrySerde(); serde.configure(List.of("wontbeused"), registryClient, "%s-key", "%s-value", true); } @ParameterizedTest @CsvSource({ "test_topic, test_topic-key, KEY", "test_topic, test_topic-value, VALUE" }) @SneakyThrows void returnsSchemaDescriptionIfSchemaRegisteredInSR(String topic, String subject, Serde.Target target) { int schemaId = registryClient.register(subject, new AvroSchema("{ \"type\": \"int\" }")); int registeredVersion = registryClient.getLatestSchemaMetadata(subject).getVersion(); var schemaOptional = serde.getSchema(topic, target); assertThat(schemaOptional).isPresent(); SchemaDescription schemaDescription = schemaOptional.get(); assertThat(schemaDescription.getSchema()) .contains( "{\"$id\":\"int\",\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"type\":\"integer\"}"); assertThat(schemaDescription.getAdditionalProperties()) .containsOnlyKeys("subject", "schemaId", "latestVersion", "type") .containsEntry("subject", subject) .containsEntry("schemaId", schemaId) .containsEntry("latestVersion", registeredVersion) .containsEntry("type", "AVRO"); } @Test void returnsEmptyDescriptorIfSchemaNotRegisteredInSR() { String topic = "test"; assertThat(serde.getSchema(topic, Serde.Target.KEY)).isEmpty(); assertThat(serde.getSchema(topic, Serde.Target.VALUE)).isEmpty(); } @Test void serializeTreatsInputAsJsonAvroSchemaPayload() throws RestClientException, IOException { AvroSchema schema = new AvroSchema( "{" + " \"type\": \"record\"," + " \"name\": \"TestAvroRecord1\"," + " \"fields\": [" + " {" + " \"name\": \"field1\"," + " \"type\": \"string\"" + " }," + " {" + " \"name\": \"field2\"," + " \"type\": \"int\"" + " }" + " ]" + "}" ); String jsonValue = "{ \"field1\":\"testStr\", \"field2\": 123 }"; String topic = "test"; int schemaId = registryClient.register(topic + "-value", schema); byte[] serialized = serde.serializer(topic, Serde.Target.VALUE).serialize(jsonValue); byte[] expected = toBytesWithMagicByteAndSchemaId(schemaId, jsonValue, schema); assertThat(serialized).isEqualTo(expected); } @Test void deserializeReturnsJsonAvroMsgJsonRepresentation() throws RestClientException, IOException { AvroSchema schema = new AvroSchema( "{" + " \"type\": \"record\"," + " \"name\": \"TestAvroRecord1\"," + " \"fields\": [" + " {" + " \"name\": \"field1\"," + " \"type\": \"string\"" + " }," + " {" + " \"name\": \"field2\"," + " \"type\": \"int\"" + " }" + " ]" + "}" ); String jsonValue = "{ \"field1\":\"testStr\", \"field2\": 123 }"; String topic = "test"; int schemaId = registryClient.register(topic + "-value", schema); byte[] data = toBytesWithMagicByteAndSchemaId(schemaId, jsonValue, schema); var result = serde.deserializer(topic, Serde.Target.VALUE).deserialize(null, data); assertJsonsEqual(jsonValue, result.getResult()); assertThat(result.getType()).isEqualTo(DeserializeResult.Type.JSON); assertThat(result.getAdditionalProperties()) .contains(Map.entry("type", "AVRO")) .contains(Map.entry("schemaId", schemaId)); } @Nested class SerdeWithDisabledSubjectExistenceCheck { @BeforeEach void init() { serde.configure(List.of("wontbeused"), registryClient, "%s-key", "%s-value", false); } @Test void canDeserializeAlwaysReturnsTrue() { String topic = RandomString.make(10); assertThat(serde.canDeserialize(topic, Serde.Target.KEY)).isTrue(); assertThat(serde.canDeserialize(topic, Serde.Target.VALUE)).isTrue(); } } @Nested class SerdeWithEnabledSubjectExistenceCheck { @BeforeEach void init() { serde.configure(List.of("wontbeused"), registryClient, "%s-key", "%s-value", true); } @Test void canDeserializeReturnsTrueIfSubjectExists() throws Exception { String topic = RandomString.make(10); registryClient.register(topic + "-key", new AvroSchema("\"int\"")); registryClient.register(topic + "-value", new AvroSchema("\"int\"")); assertThat(serde.canDeserialize(topic, Serde.Target.KEY)).isTrue(); assertThat(serde.canDeserialize(topic, Serde.Target.VALUE)).isTrue(); } @Test void canDeserializeReturnsFalseIfSubjectDoesNotExist() { String topic = RandomString.make(10); assertThat(serde.canDeserialize(topic, Serde.Target.KEY)).isFalse(); assertThat(serde.canDeserialize(topic, Serde.Target.VALUE)).isFalse(); } } @Test void canDeserializeAndCanSerializeReturnsTrueIfSubjectExists() throws Exception { String topic = RandomString.make(10); registryClient.register(topic + "-key", new AvroSchema("\"int\"")); registryClient.register(topic + "-value", new AvroSchema("\"int\"")); assertThat(serde.canSerialize(topic, Serde.Target.KEY)).isTrue(); assertThat(serde.canSerialize(topic, Serde.Target.VALUE)).isTrue(); } @Test void canSerializeReturnsFalseIfSubjectDoesNotExist() { String topic = RandomString.make(10); assertThat(serde.canSerialize(topic, Serde.Target.KEY)).isFalse(); assertThat(serde.canSerialize(topic, Serde.Target.VALUE)).isFalse(); } @SneakyThrows private void assertJsonsEqual(String expected, String actual) { var mapper = new JsonMapper(); assertThat(mapper.readTree(actual)).isEqualTo(mapper.readTree(expected)); } private byte[] toBytesWithMagicByteAndSchemaId(int schemaId, String json, AvroSchema schema) { return toBytesWithMagicByteAndSchemaId(schemaId, jsonToAvro(json, schema)); } private byte[] toBytesWithMagicByteAndSchemaId(int schemaId, byte[] body) { return ByteBuffer.allocate(1 + 4 + body.length) .put((byte) 0) .putInt(schemaId) .put(body) .array(); } @SneakyThrows private byte[] jsonToAvro(String json, AvroSchema schema) { GenericDatumWriter writer = new GenericDatumWriter<>(schema.rawSchema()); ByteArrayOutputStream output = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); writer.write(JsonAvroConversion.convertJsonToAvro(json, schema.rawSchema()), encoder); encoder.flush(); return output.toByteArray(); } @Test void avroFieldsRepresentationIsConsistentForSerializationAndDeserialization() throws Exception { AvroSchema schema = new AvroSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "f_int", "type": "int" }, { "name": "f_long", "type": "long" }, { "name": "f_string", "type": "string" }, { "name": "f_boolean", "type": "boolean" }, { "name": "f_float", "type": "float" }, { "name": "f_double", "type": "double" }, { "name": "f_enum", "type" : { "type": "enum", "name": "Suit", "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] } }, { "name": "f_map", "type": { "type": "map", "values" : "string", "default": {} } }, { "name": "f_union", "type": ["null", "string", "int" ] }, { "name": "f_optional_to_test_not_filled_case", "type": [ "null", "string"] }, { "name" : "f_fixed", "type" : { "type" : "fixed" ,"size" : 8, "name": "long_encoded" } }, { "name" : "f_bytes", "type": "bytes" } ] }""" ); String jsonPayload = """ { "f_int": 123, "f_long": 4294967294, "f_string": "string here", "f_boolean": true, "f_float": 123.1, "f_double": 123456.123456, "f_enum": "SPADES", "f_map": { "k1": "string value" }, "f_union": { "int": 123 }, "f_fixed": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0004Ò", "f_bytes": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\t)" } """; registryClient.register("test-value", schema); assertSerdeCycle("test", jsonPayload); } @Test void avroLogicalTypesRepresentationIsConsistentForSerializationAndDeserialization() throws Exception { AvroSchema schema = new AvroSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "lt_date", "type": { "type": "int", "logicalType": "date" } }, { "name": "lt_uuid", "type": { "type": "string", "logicalType": "uuid" } }, { "name": "lt_decimal", "type": { "type": "bytes", "logicalType": "decimal", "precision": 22, "scale":10 } }, { "name": "lt_time_millis", "type": { "type": "int", "logicalType": "time-millis"} }, { "name": "lt_time_micros", "type": { "type": "long", "logicalType": "time-micros"} }, { "name": "lt_timestamp_millis", "type": { "type": "long", "logicalType": "timestamp-millis" } }, { "name": "lt_timestamp_micros", "type": { "type": "long", "logicalType": "timestamp-micros" } }, { "name": "lt_local_timestamp_millis", "type": { "type": "long", "logicalType": "local-timestamp-millis" } }, { "name": "lt_local_timestamp_micros", "type": { "type": "long", "logicalType": "local-timestamp-micros" } } ] }""" ); String jsonPayload = """ { "lt_date":"1991-08-14", "lt_decimal": 2.1617413862327545E11, "lt_time_millis": "10:15:30.001", "lt_time_micros": "10:15:30.123456", "lt_uuid": "a37b75ca-097c-5d46-6119-f0637922e908", "lt_timestamp_millis": "2007-12-03T10:15:30.123Z", "lt_timestamp_micros": "2007-12-03T10:15:30.123456Z", "lt_local_timestamp_millis": "2017-12-03T10:15:30.123", "lt_local_timestamp_micros": "2017-12-03T10:15:30.123456" } """; registryClient.register("test-value", schema); assertSerdeCycle("test", jsonPayload); } // 1. serialize input json to binary // 2. deserialize from binary // 3. check that deserialized version equal to input void assertSerdeCycle(String topic, String jsonInput) { byte[] serializedBytes = serde.serializer(topic, Serde.Target.VALUE).serialize(jsonInput); var deserializedJson = serde.deserializer(topic, Serde.Target.VALUE) .deserialize(null, serializedBytes) .getResult(); assertJsonsEqual(jsonInput, deserializedJson); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/BrokerServiceTest.java ================================================ package com.provectus.kafka.ui.service; import com.provectus.kafka.ui.AbstractIntegrationTest; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import reactor.test.StepVerifier; class BrokerServiceTest extends AbstractIntegrationTest { @Autowired private BrokerService brokerService; @Autowired private ClustersStorage clustersStorage; @Test void getBrokersReturnsFilledBrokerDto() { var localCluster = clustersStorage.getClusterByName(LOCAL).get(); StepVerifier.create(brokerService.getBrokers(localCluster)) .expectNextMatches(b -> b.getId().equals(1) && b.getHost().equals(kafka.getHost()) && b.getPort().equals(kafka.getFirstMappedPort())) .verifyComplete(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ConfigTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.BrokerConfigDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.ServerStatusDTO; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; import org.springframework.test.web.reactive.server.WebTestClient; import org.testcontainers.shaded.org.awaitility.Awaitility; public class ConfigTest extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @BeforeEach void waitUntilStatsInitialized() { Awaitility.await() .atMost(Duration.ofSeconds(10)) .pollInSameThread() .until(() -> { var stats = applicationContext.getBean(StatisticsCache.class) .get(KafkaCluster.builder().name(LOCAL).build()); return stats.getStatus() == ServerStatusDTO.ONLINE; }); } @Test public void testAlterConfig() { String name = "background.threads"; Optional bc = getConfig(name); assertThat(bc.isPresent()).isTrue(); assertThat(bc.get().getValue()).isEqualTo("10"); final String newValue = "5"; webTestClient.put() .uri("/api/clusters/{clusterName}/brokers/{id}/configs/{name}", LOCAL, 1, name) .bodyValue(Map.of( "name", name, "value", newValue ) ) .exchange() .expectStatus().isOk(); Awaitility.await() .atMost(Duration.ofSeconds(10)) .pollInSameThread() .untilAsserted(() -> { Optional bcc = getConfig(name); assertThat(bcc.isPresent()).isTrue(); assertThat(bcc.get().getValue()).isEqualTo(newValue); }); } @Test public void testAlterReadonlyConfig() { String name = "log.dirs"; webTestClient.put() .uri("/api/clusters/{clusterName}/brokers/{id}/configs/{name}", LOCAL, 1, name) .bodyValue(Map.of( "name", name, "value", "/var/lib/kafka2" ) ) .exchange() .expectStatus().isBadRequest(); } private Optional getConfig(String name) { List configs = webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/{id}/configs", LOCAL, 1) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() { }) .returnResult() .getResponseBody(); return configs.stream() .filter(c -> c.getName().equals(name)) .findAny(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KafkaConfigSanitizerTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; class KafkaConfigSanitizerTest { @Test void doNothingIfEnabledPropertySetToFalse() { final var sanitizer = new KafkaConfigSanitizer(false, List.of()); assertThat(sanitizer.sanitize("password", "secret")).isEqualTo("secret"); assertThat(sanitizer.sanitize("sasl.jaas.config", "secret")).isEqualTo("secret"); assertThat(sanitizer.sanitize("database.password", "secret")).isEqualTo("secret"); } @Test void obfuscateCredentials() { final var sanitizer = new KafkaConfigSanitizer(true, List.of()); assertThat(sanitizer.sanitize("sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("consumer.sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("producer.sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("main.consumer.sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("database.password", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("basic.auth.user.info", "secret")).isEqualTo("******"); //AWS var sanitizing assertThat(sanitizer.sanitize("aws.access.key.id", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("aws.accessKeyId", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("aws.secret.access.key", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("aws.secretAccessKey", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("aws.sessionToken", "secret")).isEqualTo("******"); } @Test void notObfuscateNormalConfigs() { final var sanitizer = new KafkaConfigSanitizer(true, List.of()); assertThat(sanitizer.sanitize("security.protocol", "SASL_SSL")).isEqualTo("SASL_SSL"); final String[] bootstrapServer = new String[] {"test1:9092", "test2:9092"}; assertThat(sanitizer.sanitize("bootstrap.servers", bootstrapServer)).isEqualTo(bootstrapServer); } @Test void obfuscateCredentialsWithDefinedPatterns() { final var sanitizer = new KafkaConfigSanitizer(true, Arrays.asList("kafka.ui", ".*test.*")); assertThat(sanitizer.sanitize("consumer.kafka.ui", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("this.is.test.credentials", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("this.is.not.credential", "not.credential")) .isEqualTo("not.credential"); assertThat(sanitizer.sanitize("database.password", "no longer credential")) .isEqualTo("no longer credential"); } @Test void sanitizeConnectorConfigDoNotFailOnNullableValues() { Map originalConfig = new HashMap<>(); originalConfig.put("password", "secret"); originalConfig.put("asIs", "normal"); originalConfig.put("nullVal", null); var sanitizedConfig = new KafkaConfigSanitizer(true, List.of()) .sanitizeConnectorConfig(originalConfig); assertThat(sanitizedConfig) .hasSize(3) .containsEntry("password", "******") .containsEntry("asIs", "normal") .containsEntry("nullVal", null); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/LogDirsTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.exception.LogDirNotFoundApiException; import com.provectus.kafka.ui.exception.TopicOrPartitionNotFoundException; import com.provectus.kafka.ui.model.BrokerTopicLogdirsDTO; import com.provectus.kafka.ui.model.BrokersLogdirsDTO; import com.provectus.kafka.ui.model.ErrorResponseDTO; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.ParameterizedTypeReference; import org.springframework.test.web.reactive.server.WebTestClient; public class LogDirsTest extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @Test public void testAllBrokers() { List dirs = webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/logdirs", LOCAL) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() {}) .returnResult() .getResponseBody(); assertThat(dirs).hasSize(1); BrokersLogdirsDTO dir = dirs.get(0); assertThat(dir.getName()).isEqualTo("/var/lib/kafka/data"); assertThat(dir.getTopics().stream().anyMatch(t -> t.getName().equals("__consumer_offsets"))) .isTrue(); BrokerTopicLogdirsDTO topic = dir.getTopics().stream() .filter(t -> t.getName().equals("__consumer_offsets")) .findAny().get(); assertThat(topic.getPartitions()).hasSize(1); assertThat(topic.getPartitions().get(0).getBroker()).isEqualTo(1); assertThat(topic.getPartitions().get(0).getSize()).isPositive(); } @Test public void testOneBrokers() { List dirs = webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/logdirs?broker=1", LOCAL) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() {}) .returnResult() .getResponseBody(); assertThat(dirs).hasSize(1); BrokersLogdirsDTO dir = dirs.get(0); assertThat(dir.getName()).isEqualTo("/var/lib/kafka/data"); assertThat(dir.getTopics().stream().anyMatch(t -> t.getName().equals("__consumer_offsets"))) .isTrue(); BrokerTopicLogdirsDTO topic = dir.getTopics().stream() .filter(t -> t.getName().equals("__consumer_offsets")) .findAny().get(); assertThat(topic.getPartitions()).hasSize(1); assertThat(topic.getPartitions().get(0).getBroker()).isEqualTo(1); assertThat(topic.getPartitions().get(0).getSize()).isPositive(); } @Test public void testWrongBrokers() { List dirs = webTestClient.get() .uri("/api/clusters/{clusterName}/brokers/logdirs?broker=2", LOCAL) .exchange() .expectStatus().isOk() .expectBody(new ParameterizedTypeReference>() {}) .returnResult() .getResponseBody(); assertThat(dirs).isEmpty(); } @Test public void testChangeDirToWrongDir() { ErrorResponseDTO dirs = webTestClient.patch() .uri("/api/clusters/{clusterName}/brokers/{id}/logdirs", LOCAL, 1) .bodyValue(Map.of( "topic", "__consumer_offsets", "partition", "0", "logDir", "/asdf/as" ) ) .exchange() .expectStatus().isBadRequest() .expectBody(ErrorResponseDTO.class) .returnResult() .getResponseBody(); assertThat(dirs.getMessage()) .isEqualTo(new LogDirNotFoundApiException().getMessage()); dirs = webTestClient.patch() .uri("/api/clusters/{clusterName}/brokers/{id}/logdirs", LOCAL, 1) .bodyValue(Map.of( "topic", "asdf", "partition", "0", "logDir", "/var/lib/kafka/data" ) ) .exchange() .expectStatus().isBadRequest() .expectBody(ErrorResponseDTO.class) .returnResult() .getResponseBody(); assertThat(dirs.getMessage()) .isEqualTo(new TopicOrPartitionNotFoundException().getMessage()); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/MessagesServiceTest.java ================================================ package com.provectus.kafka.ui.service; import static com.provectus.kafka.ui.service.MessagesService.execSmartFilterTest; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.exception.TopicNotFoundException; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.CreateTopicMessageDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.SeekTypeDTO; import com.provectus.kafka.ui.model.SmartFilterTestExecutionDTO; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.producer.KafkaTestProducer; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.kafka.clients.admin.NewTopic; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; class MessagesServiceTest extends AbstractIntegrationTest { private static final String MASKED_TOPICS_PREFIX = "masking-test-"; private static final String NON_EXISTING_TOPIC = UUID.randomUUID().toString(); @Autowired MessagesService messagesService; KafkaCluster cluster; @BeforeEach void init() { cluster = applicationContext .getBean(ClustersStorage.class) .getClusterByName(LOCAL) .get(); } @Test void deleteTopicMessagesReturnsExceptionWhenTopicNotFound() { StepVerifier.create(messagesService.deleteTopicMessages(cluster, NON_EXISTING_TOPIC, List.of())) .expectError(TopicNotFoundException.class) .verify(); } @Test void sendMessageReturnsExceptionWhenTopicNotFound() { StepVerifier.create(messagesService.sendMessage(cluster, NON_EXISTING_TOPIC, new CreateTopicMessageDTO())) .expectError(TopicNotFoundException.class) .verify(); } @Test void loadMessagesReturnsExceptionWhenTopicNotFound() { StepVerifier.create(messagesService .loadMessages(cluster, NON_EXISTING_TOPIC, null, null, null, 1, null, "String", "String")) .expectError(TopicNotFoundException.class) .verify(); } @Test void maskingAppliedOnConfiguredClusters() throws Exception { String testTopic = MASKED_TOPICS_PREFIX + UUID.randomUUID(); try (var producer = KafkaTestProducer.forKafka(kafka)) { createTopic(new NewTopic(testTopic, 1, (short) 1)); producer.send(testTopic, "message1"); producer.send(testTopic, "message2").get(); Flux msgsFlux = messagesService.loadMessages( cluster, testTopic, new ConsumerPosition(SeekTypeDTO.BEGINNING, testTopic, null), null, null, 100, SeekDirectionDTO.FORWARD, StringSerde.name(), StringSerde.name() ).filter(evt -> evt.getType() == TopicMessageEventDTO.TypeEnum.MESSAGE) .map(TopicMessageEventDTO::getMessage); // both messages should be masked StepVerifier.create(msgsFlux) .expectNextMatches(msg -> msg.getContent().equals("***")) .expectNextMatches(msg -> msg.getContent().equals("***")) .verifyComplete(); } finally { deleteTopic(testTopic); } } @Test void execSmartFilterTestReturnsExecutionResult() { var params = new SmartFilterTestExecutionDTO() .filterCode("key != null && value != null && headers != null && timestampMs != null && offset != null") .key("1234") .value("{ \"some\" : \"value\" } ") .headers(Map.of("h1", "hv1")) .offset(12345L) .timestampMs(System.currentTimeMillis()) .partition(1); assertThat(execSmartFilterTest(params).getResult()).isTrue(); params.setFilterCode("return false"); assertThat(execSmartFilterTest(params).getResult()).isFalse(); } @Test void execSmartFilterTestReturnsErrorOnFilterApplyError() { var result = execSmartFilterTest( new SmartFilterTestExecutionDTO() .filterCode("return 1/0") ); assertThat(result.getResult()).isNull(); assertThat(result.getError()).containsIgnoringCase("execution error"); } @Test void execSmartFilterTestReturnsErrorOnFilterCompilationError() { var result = execSmartFilterTest( new SmartFilterTestExecutionDTO() .filterCode("this is invalid groovy syntax = 1") ); assertThat(result.getResult()).isNull(); assertThat(result.getError()).containsIgnoringCase("Compilation error"); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/OffsetsResetServiceTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.exception.NotFoundException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.model.KafkaCluster; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.BytesDeserializer; import org.apache.kafka.common.serialization.BytesSerializer; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; public class OffsetsResetServiceTest extends AbstractIntegrationTest { private static final int PARTITIONS = 5; private final String groupId = "OffsetsResetServiceTestGroup-" + UUID.randomUUID(); private final String topic = "OffsetsResetServiceTestTopic-" + UUID.randomUUID(); private KafkaCluster cluster; private OffsetsResetService offsetsResetService; @BeforeEach void init() { cluster = applicationContext.getBean(ClustersStorage.class).getClusterByName(LOCAL).get(); offsetsResetService = new OffsetsResetService(applicationContext.getBean(AdminClientService.class)); createTopic(new NewTopic(topic, PARTITIONS, (short) 1)); createConsumerGroup(); } @AfterEach void cleanUp() { deleteTopic(topic); } private void createConsumerGroup() { try (var consumer = groupConsumer()) { consumer.subscribe(Pattern.compile("no-such-topic-pattern")); consumer.poll(Duration.ofMillis(200)); consumer.commitSync(); } } @Test void failsIfGroupDoesNotExists() { List> expectedNotFound = List.of( offsetsResetService .resetToEarliest(cluster, "non-existing-group", topic, null), offsetsResetService .resetToLatest(cluster, "non-existing-group", topic, null), offsetsResetService .resetToTimestamp(cluster, "non-existing-group", topic, null, System.currentTimeMillis()), offsetsResetService .resetToOffsets(cluster, "non-existing-group", topic, Map.of()) ); for (Mono mono : expectedNotFound) { StepVerifier.create(mono) .expectErrorMatches(t -> t instanceof NotFoundException) .verify(); } } @Test void failsIfGroupIsActive() { // starting consumer to activate group try (var consumer = groupConsumer()) { consumer.subscribe(Pattern.compile("no-such-topic-pattern")); consumer.poll(Duration.ofMillis(100)); List> expectedValidationError = List.of( offsetsResetService.resetToEarliest(cluster, groupId, topic, null), offsetsResetService.resetToLatest(cluster, groupId, topic, null), offsetsResetService .resetToTimestamp(cluster, groupId, topic, null, System.currentTimeMillis()), offsetsResetService.resetToOffsets(cluster, groupId, topic, Map.of()) ); for (Mono mono : expectedValidationError) { StepVerifier.create(mono) .expectErrorMatches(t -> t instanceof ValidationException) .verify(); } } } @Test void resetToOffsets() { sendMsgsToPartition(Map.of(0, 10, 1, 10, 2, 10)); var expectedOffsets = Map.of(0, 5L, 1, 5L, 2, 5L); offsetsResetService.resetToOffsets(cluster, groupId, topic, expectedOffsets).block(); assertOffsets(expectedOffsets); } @Test void resetToOffsetsCommitsEarliestOrLatestOffsetsIfOffsetsBoundsNotValid() { sendMsgsToPartition(Map.of(0, 10, 1, 10, 2, 10)); var offsetsWithInValidBounds = Map.of(0, -2L, 1, 5L, 2, 500L); var expectedOffsets = Map.of(0, 0L, 1, 5L, 2, 10L); offsetsResetService.resetToOffsets(cluster, groupId, topic, offsetsWithInValidBounds).block(); assertOffsets(expectedOffsets); } @Test void resetToEarliest() { sendMsgsToPartition(Map.of(0, 10, 1, 10, 2, 10)); commit(Map.of(0, 5L, 1, 5L, 2, 5L)); offsetsResetService.resetToEarliest(cluster, groupId, topic, List.of(0, 1)).block(); assertOffsets(Map.of(0, 0L, 1, 0L, 2, 5L)); commit(Map.of(0, 5L, 1, 5L, 2, 5L)); offsetsResetService.resetToEarliest(cluster, groupId, topic, null).block(); assertOffsets(Map.of(0, 0L, 1, 0L, 2, 0L, 3, 0L, 4, 0L)); } @Test void resetToLatest() { sendMsgsToPartition(Map.of(0, 10, 1, 10, 2, 10, 3, 10, 4, 10)); commit(Map.of(0, 5L, 1, 5L, 2, 5L)); offsetsResetService.resetToLatest(cluster, groupId, topic, List.of(0, 1)).block(); assertOffsets(Map.of(0, 10L, 1, 10L, 2, 5L)); commit(Map.of(0, 5L, 1, 5L, 2, 5L)); offsetsResetService.resetToLatest(cluster, groupId, topic, null).block(); assertOffsets(Map.of(0, 10L, 1, 10L, 2, 10L, 3, 10L, 4, 10L)); } @Test void resetToTimestamp() { send( Stream.of( new ProducerRecord(topic, 0, 1000L, null, null), new ProducerRecord(topic, 0, 1500L, null, null), new ProducerRecord(topic, 0, 2000L, null, null), new ProducerRecord(topic, 1, 1000L, null, null), new ProducerRecord(topic, 1, 2000L, null, null), new ProducerRecord(topic, 2, 1000L, null, null), new ProducerRecord(topic, 2, 1100L, null, null), new ProducerRecord(topic, 2, 1200L, null, null))); offsetsResetService.resetToTimestamp( cluster, groupId, topic, List.of(0, 1, 2, 3), 1600L ).block(); assertOffsets(Map.of(0, 2L, 1, 1L, 2, 3L, 3, 0L)); } private void commit(Map offsetsToCommit) { try (var consumer = groupConsumer()) { consumer.commitSync( offsetsToCommit.entrySet().stream() .collect(Collectors.toMap( e -> new TopicPartition(topic, e.getKey()), e -> new OffsetAndMetadata(e.getValue()))) ); } } private void sendMsgsToPartition(Map msgsCountForPartitions) { Bytes bytes = new Bytes("noMatter".getBytes()); send( msgsCountForPartitions.entrySet().stream() .flatMap(e -> IntStream.range(0, e.getValue()) .mapToObj(i -> new ProducerRecord<>(topic, e.getKey(), bytes, bytes)))); } private void send(Stream> toSend) { var properties = new Properties(); properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); var serializer = new BytesSerializer(); try (var producer = new KafkaProducer<>(properties, serializer, serializer)) { toSend.forEach(producer::send); producer.flush(); } } private void assertOffsets(Map expectedOffsets) { try (var consumer = groupConsumer()) { var tps = expectedOffsets.keySet().stream() .map(idx -> new TopicPartition(topic, idx)) .collect(Collectors.toSet()); var actualOffsets = consumer.committed(tps).entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().partition(), e -> e.getValue().offset())); assertThat(actualOffsets).isEqualTo(expectedOffsets); } } private Consumer groupConsumer() { Properties props = new Properties(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, "kafka-ui-" + UUID.randomUUID()); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.getBootstrapServers()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); return new KafkaConsumer<>(props); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ReactiveAdminClientTest.java ================================================ package com.provectus.kafka.ui.service; import static com.provectus.kafka.ui.service.ReactiveAdminClient.toMonoWithExceptionFilter; import static java.util.Objects.requireNonNull; import static org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.ThrowableAssert.ThrowingCallable; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.producer.KafkaTestProducer; import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.function.Function; import java.util.stream.Stream; import lombok.SneakyThrows; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.OffsetSpec; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.common.serialization.StringDeserializer; import org.assertj.core.api.ThrowableAssert; import org.junit.function.ThrowingRunnable; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.test.StepVerifier; class ReactiveAdminClientTest extends AbstractIntegrationTest { private final List clearings = new ArrayList<>(); private AdminClient adminClient; private ReactiveAdminClient reactiveAdminClient; @BeforeEach void init() { AdminClientService adminClientService = applicationContext.getBean(AdminClientService.class); ClustersStorage clustersStorage = applicationContext.getBean(ClustersStorage.class); reactiveAdminClient = requireNonNull(adminClientService.get(clustersStorage.getClusterByName(LOCAL).get()).block()); adminClient = reactiveAdminClient.getClient(); } @AfterEach void tearDown() { for (ThrowingRunnable clearing : clearings) { try { clearing.run(); } catch (Throwable th) { //NOOP } } } @Test void testUpdateTopicConfigs() throws Exception { String topic = UUID.randomUUID().toString(); createTopics(new NewTopic(topic, 1, (short) 1)); var configResource = new ConfigResource(ConfigResource.Type.TOPIC, topic); adminClient.incrementalAlterConfigs( Map.of( configResource, List.of( new AlterConfigOp(new ConfigEntry("compression.type", "gzip"), AlterConfigOp.OpType.SET), new AlterConfigOp(new ConfigEntry("retention.bytes", "12345678"), AlterConfigOp.OpType.SET) ) ) ).all().get(); StepVerifier.create( reactiveAdminClient.updateTopicConfig( topic, Map.of( "compression.type", "snappy", //changing existing config "file.delete.delay.ms", "12345" // adding new one ) ) ).expectComplete().verify(); Config config = adminClient.describeConfigs(List.of(configResource)).values().get(configResource).get(); assertThat(config.get("retention.bytes").value()).isNotEqualTo("12345678"); // wes reset to default assertThat(config.get("compression.type").value()).isEqualTo("snappy"); assertThat(config.get("file.delete.delay.ms").value()).isEqualTo("12345"); } @SneakyThrows void createTopics(NewTopic... topics) { adminClient.createTopics(List.of(topics)).all().get(); clearings.add(() -> adminClient.deleteTopics(Stream.of(topics).map(NewTopic::name).toList()).all().get()); } void fillTopic(String topic, int msgsCnt) { try (var producer = KafkaTestProducer.forKafka(kafka)) { for (int i = 0; i < msgsCnt; i++) { producer.send(topic, UUID.randomUUID().toString()); } } } @Test void testToMonoWithExceptionFilter() { var failedFuture = new KafkaFutureImpl(); failedFuture.completeExceptionally(new UnknownTopicOrPartitionException()); var okFuture = new KafkaFutureImpl(); okFuture.complete("done"); var emptyFuture = new KafkaFutureImpl(); emptyFuture.complete(null); Map> arg = Map.of( "failure", failedFuture, "ok", okFuture, "empty", emptyFuture ); StepVerifier.create(toMonoWithExceptionFilter(arg, UnknownTopicOrPartitionException.class)) .assertNext(result -> assertThat(result).hasSize(1).containsEntry("ok", "done")) .verifyComplete(); } @Test void filterPartitionsWithLeaderCheckSkipsPartitionsFromTopicWhereSomePartitionsHaveNoLeader() { var filteredPartitions = ReactiveAdminClient.filterPartitionsWithLeaderCheck( List.of( // contains partitions with no leader new TopicDescription("noLeaderTopic", false, List.of( new TopicPartitionInfo(0, new Node(1, "n1", 9092), List.of(), List.of()), new TopicPartitionInfo(1, null, List.of(), List.of()))), // should be skipped by predicate new TopicDescription("skippingByPredicate", false, List.of( new TopicPartitionInfo(0, new Node(1, "n1", 9092), List.of(), List.of()))), // good topic new TopicDescription("good", false, List.of( new TopicPartitionInfo(0, new Node(1, "n1", 9092), List.of(), List.of()), new TopicPartitionInfo(1, new Node(2, "n2", 9092), List.of(), List.of())) )), p -> !p.topic().equals("skippingByPredicate"), false ); assertThat(filteredPartitions) .containsExactlyInAnyOrder( new TopicPartition("good", 0), new TopicPartition("good", 1) ); } @Test void filterPartitionsWithLeaderCheckThrowExceptionIfThereIsSomePartitionsWithoutLeaderAndFlagSet() { ThrowingCallable call = () -> ReactiveAdminClient.filterPartitionsWithLeaderCheck( List.of( // contains partitions with no leader new TopicDescription("t1", false, List.of( new TopicPartitionInfo(0, new Node(1, "n1", 9092), List.of(), List.of()), new TopicPartitionInfo(1, null, List.of(), List.of()))), new TopicDescription("t2", false, List.of( new TopicPartitionInfo(0, new Node(1, "n1", 9092), List.of(), List.of())) )), p -> true, // setting failOnNoLeader flag true ); assertThatThrownBy(call).isInstanceOf(ValidationException.class); } @Test void testListOffsetsUnsafe() { String topic = UUID.randomUUID().toString(); createTopics(new NewTopic(topic, 2, (short) 1)); // sending messages to have non-zero offsets for tp try (var producer = KafkaTestProducer.forKafka(kafka)) { producer.send(new ProducerRecord<>(topic, 1, "k", "v")); producer.send(new ProducerRecord<>(topic, 1, "k", "v")); } var requestedPartitions = List.of( new TopicPartition(topic, 0), new TopicPartition(topic, 1) ); StepVerifier.create(reactiveAdminClient.listOffsetsUnsafe(requestedPartitions, OffsetSpec.earliest())) .assertNext(offsets -> { assertThat(offsets) .hasSize(2) .containsEntry(new TopicPartition(topic, 0), 0L) .containsEntry(new TopicPartition(topic, 1), 0L); }) .verifyComplete(); StepVerifier.create(reactiveAdminClient.listOffsetsUnsafe(requestedPartitions, OffsetSpec.latest())) .assertNext(offsets -> { assertThat(offsets) .hasSize(2) .containsEntry(new TopicPartition(topic, 0), 0L) .containsEntry(new TopicPartition(topic, 1), 2L); }) .verifyComplete(); } @Test void testListConsumerGroupOffsets() throws Exception { String topic = UUID.randomUUID().toString(); String anotherTopic = UUID.randomUUID().toString(); createTopics(new NewTopic(topic, 2, (short) 1), new NewTopic(anotherTopic, 1, (short) 1)); fillTopic(topic, 10); Function> consumerSupplier = groupName -> { Properties p = new Properties(); p.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); p.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupName); p.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); p.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); p.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); p.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); return new KafkaConsumer(p); }; String fullyPolledConsumer = UUID.randomUUID().toString(); try (KafkaConsumer c = consumerSupplier.apply(fullyPolledConsumer)) { c.subscribe(List.of(topic)); int polled = 0; while (polled < 10) { polled += c.poll(Duration.ofMillis(50)).count(); } c.commitSync(); } String polled1MsgConsumer = UUID.randomUUID().toString(); try (KafkaConsumer c = consumerSupplier.apply(polled1MsgConsumer)) { c.subscribe(List.of(topic)); c.poll(Duration.ofMillis(100)); c.commitSync(Map.of(tp(topic, 0), new OffsetAndMetadata(1))); } String noCommitConsumer = UUID.randomUUID().toString(); try (KafkaConsumer c = consumerSupplier.apply(noCommitConsumer)) { c.subscribe(List.of(topic)); c.poll(Duration.ofMillis(100)); } Map endOffsets = adminClient.listOffsets(Map.of( tp(topic, 0), OffsetSpec.latest(), tp(topic, 1), OffsetSpec.latest())).all().get(); StepVerifier.create( reactiveAdminClient.listConsumerGroupOffsets( List.of(fullyPolledConsumer, polled1MsgConsumer, noCommitConsumer), List.of( tp(topic, 0), tp(topic, 1), tp(anotherTopic, 0)) ) ).assertNext(table -> { assertThat(table.row(polled1MsgConsumer)) .containsEntry(tp(topic, 0), 1L) .hasSize(1); assertThat(table.row(noCommitConsumer)) .isEmpty(); assertThat(table.row(fullyPolledConsumer)) .containsEntry(tp(topic, 0), endOffsets.get(tp(topic, 0)).offset()) .containsEntry(tp(topic, 1), endOffsets.get(tp(topic, 1)).offset()) .hasSize(2); }) .verifyComplete(); } private static TopicPartition tp(String topic, int partition) { return new TopicPartition(topic, partition); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/RecordEmitterTest.java ================================================ package com.provectus.kafka.ui.service; import static com.provectus.kafka.ui.model.SeekTypeDTO.BEGINNING; import static com.provectus.kafka.ui.model.SeekTypeDTO.LATEST; import static com.provectus.kafka.ui.model.SeekTypeDTO.OFFSET; import static com.provectus.kafka.ui.model.SeekTypeDTO.TIMESTAMP; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.emitter.BackwardEmitter; import com.provectus.kafka.ui.emitter.EnhancedConsumer; import com.provectus.kafka.ui.emitter.ForwardEmitter; import com.provectus.kafka.ui.emitter.PollingSettings; import com.provectus.kafka.ui.emitter.PollingThrottler; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.producer.KafkaTestProducer; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.serdes.ConsumerRecordDeserializer; import com.provectus.kafka.ui.serdes.PropertyResolverImpl; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import com.provectus.kafka.ui.util.ApplicationMetrics; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.header.internals.RecordHeader; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxSink; import reactor.test.StepVerifier; @Slf4j class RecordEmitterTest extends AbstractIntegrationTest { static final int PARTITIONS = 5; static final int MSGS_PER_PARTITION = 100; static final String TOPIC = RecordEmitterTest.class.getSimpleName() + "_" + UUID.randomUUID(); static final String EMPTY_TOPIC = TOPIC + "_empty"; static final List SENT_RECORDS = new ArrayList<>(); static final ConsumerRecordDeserializer RECORD_DESERIALIZER = createRecordsDeserializer(); static final Predicate NOOP_FILTER = m -> true; @BeforeAll static void generateMsgs() throws Exception { createTopic(new NewTopic(TOPIC, PARTITIONS, (short) 1)); createTopic(new NewTopic(EMPTY_TOPIC, PARTITIONS, (short) 1)); try (var producer = KafkaTestProducer.forKafka(kafka)) { for (int partition = 0; partition < PARTITIONS; partition++) { for (int i = 0; i < MSGS_PER_PARTITION; i++) { long ts = System.currentTimeMillis() + i; var value = "msg_" + partition + "_" + i; var metadata = producer.send( new ProducerRecord<>( TOPIC, partition, ts, null, value, List.of( new RecordHeader("name", null), new RecordHeader("name2", "value".getBytes()) ) ) ).get(); SENT_RECORDS.add( new Record( value, new TopicPartition(metadata.topic(), metadata.partition()), metadata.offset(), ts ) ); } } } } @AfterAll static void cleanup() { deleteTopic(TOPIC); deleteTopic(EMPTY_TOPIC); SENT_RECORDS.clear(); } private static ConsumerRecordDeserializer createRecordsDeserializer() { Serde s = new StringSerde(); s.configure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); return new ConsumerRecordDeserializer( StringSerde.name(), s.deserializer(null, Serde.Target.KEY), StringSerde.name(), s.deserializer(null, Serde.Target.VALUE), StringSerde.name(), s.deserializer(null, Serde.Target.KEY), s.deserializer(null, Serde.Target.VALUE), msg -> msg ); } @Test void pollNothingOnEmptyTopic() { var forwardEmitter = new ForwardEmitter( this::createConsumer, new ConsumerPosition(BEGINNING, EMPTY_TOPIC, null), 100, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(BEGINNING, EMPTY_TOPIC, null), 100, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); StepVerifier.create(Flux.create(forwardEmitter)) .expectNextMatches(m -> m.getType().equals(TopicMessageEventDTO.TypeEnum.PHASE)) .expectNextMatches(m -> m.getType().equals(TopicMessageEventDTO.TypeEnum.DONE)) .expectComplete() .verify(); StepVerifier.create(Flux.create(backwardEmitter)) .expectNextMatches(m -> m.getType().equals(TopicMessageEventDTO.TypeEnum.PHASE)) .expectNextMatches(m -> m.getType().equals(TopicMessageEventDTO.TypeEnum.DONE)) .expectComplete() .verify(); } @Test void pollFullTopicFromBeginning() { var forwardEmitter = new ForwardEmitter( this::createConsumer, new ConsumerPosition(BEGINNING, TOPIC, null), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(LATEST, TOPIC, null), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); List expectedValues = SENT_RECORDS.stream().map(Record::getValue).collect(Collectors.toList()); expectEmitter(forwardEmitter, expectedValues); expectEmitter(backwardEmitter, expectedValues); } @Test void pollWithOffsets() { Map targetOffsets = new HashMap<>(); for (int i = 0; i < PARTITIONS; i++) { long offset = ThreadLocalRandom.current().nextLong(MSGS_PER_PARTITION); targetOffsets.put(new TopicPartition(TOPIC, i), offset); } var forwardEmitter = new ForwardEmitter( this::createConsumer, new ConsumerPosition(OFFSET, TOPIC, targetOffsets), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(OFFSET, TOPIC, targetOffsets), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var expectedValues = SENT_RECORDS.stream() .filter(r -> r.getOffset() >= targetOffsets.get(r.getTp())) .map(Record::getValue) .collect(Collectors.toList()); expectEmitter(forwardEmitter, expectedValues); expectedValues = SENT_RECORDS.stream() .filter(r -> r.getOffset() < targetOffsets.get(r.getTp())) .map(Record::getValue) .collect(Collectors.toList()); expectEmitter(backwardEmitter, expectedValues); } @Test void pollWithTimestamps() { Map targetTimestamps = new HashMap<>(); final Map> perPartition = SENT_RECORDS.stream().collect(Collectors.groupingBy((r) -> r.tp)); for (int i = 0; i < PARTITIONS; i++) { final List records = perPartition.get(new TopicPartition(TOPIC, i)); int randRecordIdx = ThreadLocalRandom.current().nextInt(records.size()); log.info("partition: {} position: {}", i, randRecordIdx); targetTimestamps.put( new TopicPartition(TOPIC, i), records.get(randRecordIdx).getTimestamp() ); } var forwardEmitter = new ForwardEmitter( this::createConsumer, new ConsumerPosition(TIMESTAMP, TOPIC, targetTimestamps), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(TIMESTAMP, TOPIC, targetTimestamps), PARTITIONS * MSGS_PER_PARTITION, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var expectedValues = SENT_RECORDS.stream() .filter(r -> r.getTimestamp() >= targetTimestamps.get(r.getTp())) .map(Record::getValue) .collect(Collectors.toList()); expectEmitter(forwardEmitter, expectedValues); expectedValues = SENT_RECORDS.stream() .filter(r -> r.getTimestamp() < targetTimestamps.get(r.getTp())) .map(Record::getValue) .collect(Collectors.toList()); expectEmitter(backwardEmitter, expectedValues); } @Test void backwardEmitterSeekToEnd() { final int numMessages = 100; final Map targetOffsets = new HashMap<>(); for (int i = 0; i < PARTITIONS; i++) { targetOffsets.put(new TopicPartition(TOPIC, i), (long) MSGS_PER_PARTITION); } var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(OFFSET, TOPIC, targetOffsets), numMessages, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); var expectedValues = SENT_RECORDS.stream() .filter(r -> r.getOffset() < targetOffsets.get(r.getTp())) .filter(r -> r.getOffset() >= (targetOffsets.get(r.getTp()) - (numMessages / PARTITIONS))) .map(Record::getValue) .collect(Collectors.toList()); assertThat(expectedValues).size().isEqualTo(numMessages); expectEmitter(backwardEmitter, expectedValues); } @Test void backwardEmitterSeekToBegin() { Map offsets = new HashMap<>(); for (int i = 0; i < PARTITIONS; i++) { offsets.put(new TopicPartition(TOPIC, i), 0L); } var backwardEmitter = new BackwardEmitter( this::createConsumer, new ConsumerPosition(OFFSET, TOPIC, offsets), 100, RECORD_DESERIALIZER, NOOP_FILTER, PollingSettings.createDefault() ); expectEmitter(backwardEmitter, 100, e -> e.expectNextCount(0), StepVerifier.Assertions::hasNotDroppedElements ); } private void expectEmitter(Consumer> emitter, List expectedValues) { expectEmitter(emitter, expectedValues.size(), e -> e.recordWith(ArrayList::new) .expectNextCount(expectedValues.size()) .expectRecordedMatches(r -> r.containsAll(expectedValues)) .consumeRecordedWith(r -> log.info("Collected collection: {}", r)), v -> { } ); } private void expectEmitter( Consumer> emitter, int take, Function, StepVerifier.Step> stepConsumer, Consumer assertionsConsumer) { StepVerifier.FirstStep firstStep = StepVerifier.create( Flux.create(emitter) .filter(m -> m.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) .take(take) .map(m -> m.getMessage().getContent()) ); StepVerifier.Step step = stepConsumer.apply(firstStep); assertionsConsumer.accept(step.expectComplete().verifyThenAssertThat()); } private EnhancedConsumer createConsumer() { return createConsumer(Map.of()); } private EnhancedConsumer createConsumer(Map properties) { final Map map = Map.of( ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers(), ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString(), ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 19 // to check multiple polls ); Properties props = new Properties(); props.putAll(map); props.putAll(properties); return new EnhancedConsumer(props, PollingThrottler.noop(), ApplicationMetrics.noop()); } @Value static class Record { String value; TopicPartition tp; long offset; long timestamp; } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SchemaRegistryPaginationTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.controller.SchemasController; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.SchemaSubjectDTO; import com.provectus.kafka.ui.service.audit.AuditService; import com.provectus.kafka.ui.sr.model.Compatibility; import com.provectus.kafka.ui.sr.model.SchemaSubject; import com.provectus.kafka.ui.util.AccessControlServiceMock; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.stream.IntStream; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; public class SchemaRegistryPaginationTest { private static final String LOCAL_KAFKA_CLUSTER_NAME = "local"; private SchemasController controller; private void init(List subjects) { ClustersStorage clustersStorage = mock(ClustersStorage.class); when(clustersStorage.getClusterByName(isA(String.class))) .thenReturn(Optional.of(buildKafkaCluster(LOCAL_KAFKA_CLUSTER_NAME))); SchemaRegistryService schemaRegistryService = mock(SchemaRegistryService.class); when(schemaRegistryService.getAllSubjectNames(isA(KafkaCluster.class))) .thenReturn(Mono.just(subjects)); when(schemaRegistryService .getAllLatestVersionSchemas(isA(KafkaCluster.class), anyList())).thenCallRealMethod(); when(schemaRegistryService.getLatestSchemaVersionBySubject(isA(KafkaCluster.class), isA(String.class))) .thenAnswer(a -> Mono.just( new SchemaRegistryService.SubjectWithCompatibilityLevel( new SchemaSubject().subject(a.getArgument(1)), Compatibility.FULL))); this.controller = new SchemasController(schemaRegistryService); this.controller.setAccessControlService(new AccessControlServiceMock().getMock()); this.controller.setAuditService(mock(AuditService.class)); this.controller.setClustersStorage(clustersStorage); } @Test void shouldListFirst25andThen10Schemas() { init( IntStream.rangeClosed(1, 100) .boxed() .map(num -> "subject" + num) .toList() ); var schemasFirst25 = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, null).block(); assertThat(schemasFirst25.getBody().getPageCount()).isEqualTo(4); assertThat(schemasFirst25.getBody().getSchemas()).hasSize(25); assertThat(schemasFirst25.getBody().getSchemas()) .isSortedAccordingTo(Comparator.comparing(SchemaSubjectDTO::getSubject)); var schemasFirst10 = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, null, 10, null, null).block(); assertThat(schemasFirst10.getBody().getPageCount()).isEqualTo(10); assertThat(schemasFirst10.getBody().getSchemas()).hasSize(10); assertThat(schemasFirst10.getBody().getSchemas()) .isSortedAccordingTo(Comparator.comparing(SchemaSubjectDTO::getSubject)); } @Test void shouldListSchemasContaining_1() { init( IntStream.rangeClosed(1, 100) .boxed() .map(num -> "subject" + num) .toList() ); var schemasSearch7 = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, null, null, "1", null).block(); assertThat(schemasSearch7.getBody().getPageCount()).isEqualTo(1); assertThat(schemasSearch7.getBody().getSchemas()).hasSize(20); } @Test void shouldCorrectlyHandleNonPositivePageNumberAndPageSize() { init( IntStream.rangeClosed(1, 100) .boxed() .map(num -> "subject" + num) .toList() ); var schemas = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, 0, -1, null, null).block(); assertThat(schemas.getBody().getPageCount()).isEqualTo(4); assertThat(schemas.getBody().getSchemas()).hasSize(25); assertThat(schemas.getBody().getSchemas()).isSortedAccordingTo(Comparator.comparing(SchemaSubjectDTO::getSubject)); } @Test void shouldCalculateCorrectPageCountForNonDivisiblePageSize() { init( IntStream.rangeClosed(1, 100) .boxed() .map(num -> "subject" + num) .toList() ); var schemas = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, 4, 33, null, null).block(); assertThat(schemas.getBody().getPageCount()).isEqualTo(4); assertThat(schemas.getBody().getSchemas()).hasSize(1); assertThat(schemas.getBody().getSchemas().get(0).getSubject()).isEqualTo("subject99"); } private KafkaCluster buildKafkaCluster(String clusterName) { return KafkaCluster.builder() .name(clusterName) .schemaRegistryClient(mock(ReactiveFailover.class)) .build(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/SendAndReadTests.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.ObjectMapper; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.ConsumerPosition; import com.provectus.kafka.ui.model.CreateTopicMessageDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.SeekDirectionDTO; import com.provectus.kafka.ui.model.SeekTypeDTO; import com.provectus.kafka.ui.model.TopicMessageDTO; import com.provectus.kafka.ui.model.TopicMessageEventDTO; import com.provectus.kafka.ui.serdes.builtin.Int32Serde; import com.provectus.kafka.ui.serdes.builtin.Int64Serde; import com.provectus.kafka.ui.serdes.builtin.StringSerde; import com.provectus.kafka.ui.serdes.builtin.sr.SchemaRegistrySerde; import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import io.confluent.kafka.schemaregistry.json.JsonSchema; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import java.time.Duration; import java.util.Map; import java.util.Objects; import java.util.UUID; import java.util.function.Consumer; import lombok.SneakyThrows; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.TopicPartition; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import reactor.test.StepVerifier; public class SendAndReadTests extends AbstractIntegrationTest { private static final AvroSchema AVRO_SCHEMA_1 = new AvroSchema( "{" + " \"type\": \"record\"," + " \"name\": \"TestAvroRecord1\"," + " \"fields\": [" + " {" + " \"name\": \"field1\"," + " \"type\": \"string\"" + " }," + " {" + " \"name\": \"field2\"," + " \"type\": \"int\"" + " }" + " ]" + "}" ); private static final AvroSchema AVRO_SCHEMA_2 = new AvroSchema( "{" + " \"type\": \"record\"," + " \"name\": \"TestAvroRecord2\"," + " \"fields\": [" + " {" + " \"name\": \"f1\"," + " \"type\": \"int\"" + " }," + " {" + " \"name\": \"f2\"," + " \"type\": \"string\"" + " }" + " ]" + "}" ); private static final AvroSchema AVRO_SCHEMA_PRIMITIVE_STRING = new AvroSchema("{ \"type\": \"string\" }"); private static final AvroSchema AVRO_SCHEMA_PRIMITIVE_INT = new AvroSchema("{ \"type\": \"int\" }"); private static final String AVRO_SCHEMA_1_JSON_RECORD = "{ \"field1\":\"testStr\", \"field2\": 123 }"; private static final String AVRO_SCHEMA_2_JSON_RECORD = "{ \"f1\": 111, \"f2\": \"testStr\" }"; private static final ProtobufSchema PROTOBUF_SCHEMA = new ProtobufSchema( "syntax = \"proto3\";\n" + "package com.provectus;\n" + "\n" + "message TestProtoRecord {\n" + " string f1 = 1;\n" + " int32 f2 = 2;\n" + "}\n" + "\n" ); private static final String PROTOBUF_SCHEMA_JSON_RECORD = "{ \"f1\" : \"test str\", \"f2\" : 123 }"; private static final JsonSchema JSON_SCHEMA = new JsonSchema( "{ " + " \"$schema\": \"http://json-schema.org/draft-07/schema#\", " + " \"$id\": \"http://example.com/myURI.schema.json\", " + " \"title\": \"TestRecord\"," + " \"type\": \"object\"," + " \"additionalProperties\": false," + " \"properties\": {" + " \"f1\": {" + " \"type\": \"integer\"" + " }," + " \"f2\": {" + " \"type\": \"string\"" + " }," // it is important special case since there is code in KafkaJsonSchemaSerializer // that checks fields with this name (it should be worked around) + " \"schema\": {" + " \"type\": \"string\"" + " }" + " }" + "}" ); private static final String JSON_SCHEMA_RECORD = "{ \"f1\": 12, \"f2\": \"testJsonSchema1\", \"schema\": \"some txt\" }"; private KafkaCluster targetCluster; @Autowired private MessagesService messagesService; @Autowired private ClustersStorage clustersStorage; @BeforeEach void init() { targetCluster = clustersStorage.getClusterByName(LOCAL).get(); } @Test void noSchemaStringKeyStringValue() { new SendAndReadSpec() .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") .keySerde(StringSerde.name()) .content("testValue") .valueSerde(StringSerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); assertThat(polled.getContent()).isEqualTo("testValue"); }); } @Test void keyIsIntValueIsLong() { new SendAndReadSpec() .withMsgToSend( new CreateTopicMessageDTO() .key("123") .keySerde(Int32Serde.name()) .content("21474836470") .valueSerde(Int64Serde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("123"); assertThat(polled.getContent()).isEqualTo("21474836470"); }); } @Test void keyIsNull() { new SendAndReadSpec() .withMsgToSend( new CreateTopicMessageDTO() .key(null) .keySerde(StringSerde.name()) .content("testValue") .valueSerde(StringSerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isNull(); assertThat(polled.getContent()).isEqualTo("testValue"); }); } @Test void valueIsNull() { new SendAndReadSpec() .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") .keySerde(StringSerde.name()) .content(null) .valueSerde(StringSerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); assertThat(polled.getContent()).isNull(); }); } @Test void primitiveAvroSchemas() { new SendAndReadSpec() .withKeySchema(AVRO_SCHEMA_PRIMITIVE_STRING) .withValueSchema(AVRO_SCHEMA_PRIMITIVE_INT) .withMsgToSend( new CreateTopicMessageDTO() .key("\"some string\"") .keySerde(SchemaRegistrySerde.name()) .content("123") .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("\"some string\""); assertThat(polled.getContent()).isEqualTo("123"); }); } @Test void recordAvroSchema() { new SendAndReadSpec() .withKeySchema(AVRO_SCHEMA_1) .withValueSchema(AVRO_SCHEMA_2) .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(AVRO_SCHEMA_2_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); assertJsonEqual(polled.getContent(), AVRO_SCHEMA_2_JSON_RECORD); }); } @Test void keyWithNoSchemaValueWithProtoSchema() { new SendAndReadSpec() .withValueSchema(PROTOBUF_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") .keySerde(StringSerde.name()) .content(PROTOBUF_SCHEMA_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); assertJsonEqual(polled.getContent(), PROTOBUF_SCHEMA_JSON_RECORD); }); } @Test void keyWithAvroSchemaValueWithAvroSchemaKeyIsNull() { new SendAndReadSpec() .withKeySchema(AVRO_SCHEMA_1) .withValueSchema(AVRO_SCHEMA_2) .withMsgToSend( new CreateTopicMessageDTO() .key(null) .keySerde(SchemaRegistrySerde.name()) .content(AVRO_SCHEMA_2_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertThat(polled.getKey()).isNull(); assertJsonEqual(polled.getContent(), AVRO_SCHEMA_2_JSON_RECORD); }); } @Test void valueWithAvroSchemaShouldThrowExceptionIfArgIsNotValidJsonObject() { new SendAndReadSpec() .withValueSchema(AVRO_SCHEMA_2) .withMsgToSend( new CreateTopicMessageDTO() .keySerde(StringSerde.name()) // f2 has type int instead of string .content("{ \"f1\": 111, \"f2\": 123 }") .valueSerde(SchemaRegistrySerde.name()) ) .assertSendThrowsException(); } @Test void keyWithAvroSchemaValueWithProtoSchema() { new SendAndReadSpec() .withKeySchema(AVRO_SCHEMA_1) .withValueSchema(PROTOBUF_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(PROTOBUF_SCHEMA_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); assertJsonEqual(polled.getContent(), PROTOBUF_SCHEMA_JSON_RECORD); }); } @Test void valueWithProtoSchemaShouldThrowExceptionArgIsNotValidJsonObject() { new SendAndReadSpec() .withValueSchema(PROTOBUF_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(null) .keySerde(StringSerde.name()) // f2 field has type object instead of int .content("{ \"f1\" : \"test str\", \"f2\" : {} }") .valueSerde(SchemaRegistrySerde.name()) ) .assertSendThrowsException(); } @Test void keyWithProtoSchemaValueWithJsonSchema() { new SendAndReadSpec() .withKeySchema(PROTOBUF_SCHEMA) .withValueSchema(JSON_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(PROTOBUF_SCHEMA_JSON_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(JSON_SCHEMA_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), PROTOBUF_SCHEMA_JSON_RECORD); assertJsonEqual(polled.getContent(), JSON_SCHEMA_RECORD); }); } @Test void valueWithJsonSchemaThrowsExceptionIfArgIsNotValidJsonObject() { new SendAndReadSpec() .withValueSchema(JSON_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(null) .keySerde(StringSerde.name()) // 'f2' field has has type object instead of string .content("{ \"f1\": 12, \"f2\": {}, \"schema\": \"some txt\" }") .valueSerde(SchemaRegistrySerde.name()) ) .assertSendThrowsException(); } @Test void topicMessageMetadataAvro() { new SendAndReadSpec() .withKeySchema(AVRO_SCHEMA_1) .withValueSchema(AVRO_SCHEMA_2) .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(AVRO_SCHEMA_2_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); assertJsonEqual(polled.getContent(), AVRO_SCHEMA_2_JSON_RECORD); assertThat(polled.getKeySize()).isEqualTo(15L); assertThat(polled.getValueSize()).isEqualTo(15L); assertThat(polled.getKeyDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getKeyDeserializeProperties().get("type")).isEqualTo("AVRO"); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getValueDeserializeProperties().get("type")).isEqualTo("AVRO"); }); } @Test void topicMessageMetadataProtobuf() { new SendAndReadSpec() .withKeySchema(PROTOBUF_SCHEMA) .withValueSchema(PROTOBUF_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(PROTOBUF_SCHEMA_JSON_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(PROTOBUF_SCHEMA_JSON_RECORD) .valueSerde(SchemaRegistrySerde.name()) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), PROTOBUF_SCHEMA_JSON_RECORD); assertJsonEqual(polled.getContent(), PROTOBUF_SCHEMA_JSON_RECORD); assertThat(polled.getKeySize()).isEqualTo(18L); assertThat(polled.getValueSize()).isEqualTo(18L); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getKeyDeserializeProperties().get("type")).isEqualTo("PROTOBUF"); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getValueDeserializeProperties().get("type")).isEqualTo("PROTOBUF"); }); } @Test void topicMessageMetadataJson() { new SendAndReadSpec() .withKeySchema(JSON_SCHEMA) .withValueSchema(JSON_SCHEMA) .withMsgToSend( new CreateTopicMessageDTO() .key(JSON_SCHEMA_RECORD) .keySerde(SchemaRegistrySerde.name()) .content(JSON_SCHEMA_RECORD) .valueSerde(SchemaRegistrySerde.name()) .headers(Map.of("header1", "value1")) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), JSON_SCHEMA_RECORD); assertJsonEqual(polled.getContent(), JSON_SCHEMA_RECORD); assertThat(polled.getKeySize()).isEqualTo(57L); assertThat(polled.getValueSize()).isEqualTo(57L); assertThat(polled.getHeadersSize()).isEqualTo(13L); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getKeyDeserializeProperties().get("type")).isEqualTo("JSON"); assertThat(polled.getValueDeserializeProperties().get("schemaId")).isNotNull(); assertThat(polled.getValueDeserializeProperties().get("type")).isEqualTo("JSON"); }); } @Test void noKeyAndNoContentPresentTest() { new SendAndReadSpec() .withMsgToSend( new CreateTopicMessageDTO() .key(null) .keySerde(StringSerde.name()) // any serde .content(null) .valueSerde(StringSerde.name()) // any serde ) .doAssert(polled -> { assertThat(polled.getKey()).isNull(); assertThat(polled.getContent()).isNull(); }); } @SneakyThrows private void assertJsonEqual(String actual, String expected) { var mapper = new ObjectMapper(); assertThat(mapper.readTree(actual)).isEqualTo(mapper.readTree(expected)); } class SendAndReadSpec { CreateTopicMessageDTO msgToSend; ParsedSchema keySchema; ParsedSchema valueSchema; public SendAndReadSpec withMsgToSend(CreateTopicMessageDTO msg) { this.msgToSend = msg; return this; } public SendAndReadSpec withKeySchema(ParsedSchema keyScheam) { this.keySchema = keyScheam; return this; } public SendAndReadSpec withValueSchema(ParsedSchema valueSchema) { this.valueSchema = valueSchema; return this; } @SneakyThrows private String createTopicAndCreateSchemas() { Objects.requireNonNull(msgToSend); String topic = UUID.randomUUID().toString(); createTopic(new NewTopic(topic, 1, (short) 1)); if (keySchema != null) { schemaRegistry.schemaRegistryClient().register(topic + "-key", keySchema); } if (valueSchema != null) { schemaRegistry.schemaRegistryClient().register(topic + "-value", valueSchema); } return topic; } public void assertSendThrowsException() { String topic = createTopicAndCreateSchemas(); try { StepVerifier.create( messagesService.sendMessage(targetCluster, topic, msgToSend) ).expectError().verify(); } finally { deleteTopic(topic); } } @SneakyThrows public void doAssert(Consumer msgAssert) { String topic = createTopicAndCreateSchemas(); try { messagesService.sendMessage(targetCluster, topic, msgToSend).block(); TopicMessageDTO polled = messagesService.loadMessages( targetCluster, topic, new ConsumerPosition( SeekTypeDTO.BEGINNING, topic, Map.of(new TopicPartition(topic, 0), 0L) ), null, null, 1, SeekDirectionDTO.FORWARD, msgToSend.getKeySerde().get(), msgToSend.getValueSerde().get() ).filter(e -> e.getType().equals(TopicMessageEventDTO.TypeEnum.MESSAGE)) .map(TopicMessageEventDTO::getMessage) .blockLast(Duration.ofSeconds(5000)); assertThat(polled).isNotNull(); assertThat(polled.getPartition()).isEqualTo(0); assertThat(polled.getOffset()).isNotNull(); msgAssert.accept(polled); } finally { deleteTopic(topic); } } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/TopicsServicePaginationTest.java ================================================ package com.provectus.kafka.ui.service; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.controller.TopicsController; import com.provectus.kafka.ui.mapper.ClusterMapper; import com.provectus.kafka.ui.mapper.ClusterMapperImpl; import com.provectus.kafka.ui.model.InternalLogDirStats; import com.provectus.kafka.ui.model.InternalPartitionsOffsets; import com.provectus.kafka.ui.model.InternalTopic; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Metrics; import com.provectus.kafka.ui.model.SortOrderDTO; import com.provectus.kafka.ui.model.TopicColumnsToSortDTO; import com.provectus.kafka.ui.model.TopicDTO; import com.provectus.kafka.ui.service.analyze.TopicAnalysisService; import com.provectus.kafka.ui.service.audit.AuditService; import com.provectus.kafka.ui.service.rbac.AccessControlService; import com.provectus.kafka.ui.util.AccessControlServiceMock; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.TopicPartitionInfo; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; class TopicsServicePaginationTest { private static final String LOCAL_KAFKA_CLUSTER_NAME = "local"; private final TopicsService topicsService = mock(TopicsService.class); private final ClustersStorage clustersStorage = mock(ClustersStorage.class); private final ClusterMapper clusterMapper = new ClusterMapperImpl(); private final AccessControlService accessControlService = new AccessControlServiceMock().getMock(); private final TopicsController topicsController = new TopicsController(topicsService, mock(TopicAnalysisService.class), clusterMapper); private void init(Map topicsInCache) { when(clustersStorage.getClusterByName(isA(String.class))) .thenReturn(Optional.of(buildKafkaCluster(LOCAL_KAFKA_CLUSTER_NAME))); when(topicsService.getTopicsForPagination(isA(KafkaCluster.class))) .thenReturn(Mono.just(new ArrayList<>(topicsInCache.values()))); when(topicsService.loadTopics(isA(KafkaCluster.class), anyList())) .thenAnswer(a -> { List lst = a.getArgument(1); return Mono.just(lst.stream().map(topicsInCache::get).collect(Collectors.toList())); }); topicsController.setAccessControlService(accessControlService); topicsController.setAuditService(mock(AuditService.class)); topicsController.setClustersStorage(clustersStorage); } @Test public void shouldListFirst25Topics() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, null, null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(25); assertThat(topics.getBody().getTopics()) .isSortedAccordingTo(Comparator.comparing(TopicDTO::getName)); } private KafkaCluster buildKafkaCluster(String clusterName) { return KafkaCluster.builder() .name(clusterName) .build(); } @Test public void shouldListFirst25TopicsSortedByNameDescendingOrder() { var internalTopics = IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, null, TopicColumnsToSortDTO.NAME, SortOrderDTO.DESC, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(25); assertThat(topics.getBody().getTopics()).isSortedAccordingTo(Comparator.comparing(TopicDTO::getName).reversed()); assertThat(topics.getBody().getTopics()).containsExactlyElementsOf( internalTopics.values().stream() .map(clusterMapper::toTopic) .sorted(Comparator.comparing(TopicDTO::getName).reversed()) .limit(25) .collect(Collectors.toList()) ); } @Test public void shouldCalculateCorrectPageCountForNonDivisiblePageSize() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, 4, 33, null, null, null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(1); assertThat(topics.getBody().getTopics().get(0).getName()).isEqualTo("99"); } @Test public void shouldCorrectlyHandleNonPositivePageNumberAndPageSize() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, 0, -1, null, null, null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(25); assertThat(topics.getBody().getTopics()).isSortedAccordingTo(Comparator.comparing(TopicDTO::getName)); } @Test public void shouldListBotInternalAndNonInternalTopics() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 10 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, 0, -1, true, null, null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(25); assertThat(topics.getBody().getTopics()).isSortedAccordingTo(Comparator.comparing(TopicDTO::getName)); } @Test public void shouldListOnlyNonInternalTopics() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, Integer.parseInt(name) % 5 == 0, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, 4, -1, false, null, null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(4); assertThat(topics.getBody().getTopics()).hasSize(5); assertThat(topics.getBody().getTopics()).isSortedAccordingTo(Comparator.comparing(TopicDTO::getName)); } @Test public void shouldListOnlyTopicsContainingOne() { init( IntStream.rangeClosed(1, 100).boxed() .map(Objects::toString) .map(name -> new TopicDescription(name, false, List.of())) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), null, Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())) ); var topics = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, "1", null, null, null).block(); assertThat(topics.getBody().getPageCount()).isEqualTo(1); assertThat(topics.getBody().getTopics()).hasSize(20); assertThat(topics.getBody().getTopics()).isSortedAccordingTo(Comparator.comparing(TopicDTO::getName)); } @Test public void shouldListTopicsOrderedByPartitionsCount() { Map internalTopics = IntStream.rangeClosed(1, 100).boxed() .map(i -> new TopicDescription(UUID.randomUUID().toString(), false, IntStream.range(0, i) .mapToObj(p -> new TopicPartitionInfo(p, null, List.of(), List.of())) .collect(Collectors.toList()))) .map(topicDescription -> InternalTopic.from(topicDescription, List.of(), InternalPartitionsOffsets.empty(), Metrics.empty(), InternalLogDirStats.empty(), "_")) .collect(Collectors.toMap(InternalTopic::getName, Function.identity())); init(internalTopics); var topicsSortedAsc = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, null, TopicColumnsToSortDTO.TOTAL_PARTITIONS, null, null).block(); assertThat(topicsSortedAsc.getBody().getPageCount()).isEqualTo(4); assertThat(topicsSortedAsc.getBody().getTopics()).hasSize(25); assertThat(topicsSortedAsc.getBody().getTopics()).containsExactlyElementsOf( internalTopics.values().stream() .map(clusterMapper::toTopic) .sorted(Comparator.comparing(TopicDTO::getPartitionCount)) .limit(25) .collect(Collectors.toList()) ); var topicsSortedDesc = topicsController .getTopics(LOCAL_KAFKA_CLUSTER_NAME, null, null, null, null, TopicColumnsToSortDTO.TOTAL_PARTITIONS, SortOrderDTO.DESC, null).block(); assertThat(topicsSortedDesc.getBody().getPageCount()).isEqualTo(4); assertThat(topicsSortedDesc.getBody().getTopics()).hasSize(25); assertThat(topicsSortedDesc.getBody().getTopics()).containsExactlyElementsOf( internalTopics.values().stream() .map(clusterMapper::toTopic) .sorted(Comparator.comparing(TopicDTO::getPartitionCount).reversed()) .limit(25) .collect(Collectors.toList()) ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclCsvTest.java ================================================ package com.provectus.kafka.ui.service.acl; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.provectus.kafka.ui.exception.ValidationException; import java.util.Collection; import java.util.List; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; class AclCsvTest { private static final List TEST_BINDINGS = List.of( new AclBinding( new ResourcePattern(ResourceType.TOPIC, "*", PatternType.LITERAL), new AccessControlEntry("User:test1", "*", AclOperation.READ, AclPermissionType.ALLOW)), new AclBinding( new ResourcePattern(ResourceType.GROUP, "group1", PatternType.PREFIXED), new AccessControlEntry("User:test2", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)) ); @ParameterizedTest @ValueSource(strings = { "Principal,ResourceType, PatternType, ResourceName,Operation,PermissionType,Host\n" + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + "User:test2,GROUP,PREFIXED,group1,DESCRIBE,DENY,localhost", //without header "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + "\n" + "User:test2,GROUP,PREFIXED,group1,DESCRIBE,DENY,localhost" + "\n" }) void parsesValidInputCsv(String csvString) { Collection parsed = AclCsv.parseCsv(csvString); assertThat(parsed).containsExactlyInAnyOrderElementsOf(TEST_BINDINGS); } @ParameterizedTest @ValueSource(strings = { // columns > 7 "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*,1,2,3,4", // columns < 7 "User:test1,TOPIC,LITERAL,*", // enum values are illegal "User:test1,ILLEGAL,LITERAL,*,READ,ALLOW,*", "User:test1,TOPIC,LITERAL,*,READ,ILLEGAL,*" }) void throwsExceptionForInvalidInputCsv(String csvString) { assertThatThrownBy(() -> AclCsv.parseCsv(csvString)) .isInstanceOf(ValidationException.class); } @Test void transformAndParseUseSameFormat() { String csv = AclCsv.transformToCsvString(TEST_BINDINGS); Collection parsedBindings = AclCsv.parseCsv(csv); assertThat(parsedBindings).containsExactlyInAnyOrderElementsOf(TEST_BINDINGS); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/acl/AclsServiceTest.java ================================================ package com.provectus.kafka.ui.service.acl; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.model.CreateConsumerAclDTO; import com.provectus.kafka.ui.model.CreateProducerAclDTO; import com.provectus.kafka.ui.model.CreateStreamAppAclDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.AdminClientService; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.util.Collection; import java.util.List; import java.util.UUID; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.Resource; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import reactor.core.publisher.Mono; class AclsServiceTest { private static final KafkaCluster CLUSTER = KafkaCluster.builder().build(); private final ReactiveAdminClient adminClientMock = mock(ReactiveAdminClient.class); private final AdminClientService adminClientService = mock(AdminClientService.class); private final AclsService aclsService = new AclsService(adminClientService); @BeforeEach void initMocks() { when(adminClientService.get(CLUSTER)).thenReturn(Mono.just(adminClientMock)); } @Test void testSyncAclWithAclCsv() { var existingBinding1 = new AclBinding( new ResourcePattern(ResourceType.TOPIC, "*", PatternType.LITERAL), new AccessControlEntry("User:test1", "*", AclOperation.READ, AclPermissionType.ALLOW)); var existingBinding2 = new AclBinding( new ResourcePattern(ResourceType.GROUP, "group1", PatternType.PREFIXED), new AccessControlEntry("User:test2", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)); var newBindingToBeAdded = new AclBinding( new ResourcePattern(ResourceType.GROUP, "groupNew", PatternType.PREFIXED), new AccessControlEntry("User:test3", "localhost", AclOperation.DESCRIBE, AclPermissionType.DENY)); when(adminClientMock.listAcls(ResourcePatternFilter.ANY)) .thenReturn(Mono.just(List.of(existingBinding1, existingBinding2))); ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); ArgumentCaptor> deletedCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.deleteAcls(deletedCaptor.capture())) .thenReturn(Mono.empty()); aclsService.syncAclWithAclCsv( CLUSTER, "Principal,ResourceType, PatternType, ResourceName,Operation,PermissionType,Host\n" + "User:test1,TOPIC,LITERAL,*,READ,ALLOW,*\n" + "User:test3,GROUP,PREFIXED,groupNew,DESCRIBE,DENY,localhost" ).block(); Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(1) .contains(newBindingToBeAdded); Collection deletedBindings = deletedCaptor.getValue(); assertThat(deletedBindings) .hasSize(1) .contains(existingBinding2); } @Test void createsConsumerDependantAcls() { ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); var principal = UUID.randomUUID().toString(); var host = UUID.randomUUID().toString(); aclsService.createConsumerAcl( CLUSTER, new CreateConsumerAclDTO() .principal(principal) .host(host) .consumerGroups(List.of("cg1", "cg2")) .topics(List.of("t1", "t2")) ).block(); //Read, Describe on topics, Read on consumerGroups Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(6) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t2", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t2", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.GROUP, "cg1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.GROUP, "cg2", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))); } @Test void createsConsumerDependantAclsWhenTopicsAndGroupsSpecifiedByPrefix() { ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); var principal = UUID.randomUUID().toString(); var host = UUID.randomUUID().toString(); aclsService.createConsumerAcl( CLUSTER, new CreateConsumerAclDTO() .principal(principal) .host(host) .consumerGroupsPrefix("cgPref") .topicsPrefix("topicPref") ).block(); //Read, Describe on topics, Read on consumerGroups Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(3) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "topicPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "topicPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.GROUP, "cgPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))); } @Test void createsProducerDependantAcls() { ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); var principal = UUID.randomUUID().toString(); var host = UUID.randomUUID().toString(); aclsService.createProducerAcl( CLUSTER, new CreateProducerAclDTO() .principal(principal) .host(host) .topics(List.of("t1")) .idempotent(true) .transactionalId("txId1") ).block(); //Write, Describe, Create permission on topics, Write, Describe on transactionalIds //IDEMPOTENT_WRITE on cluster if idempotent is enabled (true) Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(6) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.CREATE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TRANSACTIONAL_ID, "txId1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TRANSACTIONAL_ID, "txId1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.CLUSTER, Resource.CLUSTER_NAME, PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.IDEMPOTENT_WRITE, AclPermissionType.ALLOW))); } @Test void createsProducerDependantAclsWhenTopicsAndTxIdSpecifiedByPrefix() { ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); var principal = UUID.randomUUID().toString(); var host = UUID.randomUUID().toString(); aclsService.createProducerAcl( CLUSTER, new CreateProducerAclDTO() .principal(principal) .host(host) .topicsPrefix("topicPref") .transactionsIdPrefix("txIdPref") .idempotent(false) ).block(); //Write, Describe, Create permission on topics, Write, Describe on transactionalIds //IDEMPOTENT_WRITE on cluster if idempotent is enabled (false) Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(5) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "topicPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "topicPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "topicPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.CREATE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TRANSACTIONAL_ID, "txIdPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TRANSACTIONAL_ID, "txIdPref", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.DESCRIBE, AclPermissionType.ALLOW))); } @Test void createsStreamAppDependantAcls() { ArgumentCaptor> createdCaptor = ArgumentCaptor.forClass(Collection.class); when(adminClientMock.createAcls(createdCaptor.capture())) .thenReturn(Mono.empty()); var principal = UUID.randomUUID().toString(); var host = UUID.randomUUID().toString(); aclsService.createStreamAppAcl( CLUSTER, new CreateStreamAppAclDTO() .principal(principal) .host(host) .inputTopics(List.of("t1")) .outputTopics(List.of("t2", "t3")) .applicationId("appId1") ).block(); // Read on input topics, Write on output topics // ALL on applicationId-prefixed Groups and Topics Collection createdBindings = createdCaptor.getValue(); assertThat(createdBindings) .hasSize(5) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t1", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.READ, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t2", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "t3", PatternType.LITERAL), new AccessControlEntry(principal, host, AclOperation.WRITE, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.GROUP, "appId1", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.ALL, AclPermissionType.ALLOW))) .contains(new AclBinding( new ResourcePattern(ResourceType.TOPIC, "appId1", PatternType.PREFIXED), new AccessControlEntry(principal, host, AclOperation.ALL, AclPermissionType.ALLOW))); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisServiceTest.java ================================================ package com.provectus.kafka.ui.service.analyze; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.producer.KafkaTestProducer; import com.provectus.kafka.ui.service.ClustersStorage; import java.time.Duration; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.testcontainers.shaded.org.awaitility.Awaitility; class TopicAnalysisServiceTest extends AbstractIntegrationTest { @Autowired private ClustersStorage clustersStorage; @Autowired private TopicAnalysisService topicAnalysisService; @Test void savesResultWhenAnalysisIsCompleted() { String topic = "analyze_test_" + UUID.randomUUID(); createTopic(new NewTopic(topic, 2, (short) 1)); fillTopic(topic, 1_000); var cluster = clustersStorage.getClusterByName(LOCAL).get(); topicAnalysisService.analyze(cluster, topic).block(); Awaitility.await() .atMost(Duration.ofSeconds(20)) .untilAsserted(() -> { assertThat(topicAnalysisService.getTopicAnalysis(cluster, topic)) .hasValueSatisfying(state -> { assertThat(state.getProgress()).isNull(); assertThat(state.getResult()).isNotNull(); var completedAnalyze = state.getResult(); assertThat(completedAnalyze.getTotalStats().getTotalMsgs()).isEqualTo(1_000); assertThat(completedAnalyze.getPartitionStats().size()).isEqualTo(2); }); }); } private void fillTopic(String topic, int cnt) { try (var producer = KafkaTestProducer.forKafka(kafka)) { for (int i = 0; i < cnt; i++) { producer.send( new ProducerRecord<>( topic, RandomStringUtils.randomAlphabetic(5), RandomStringUtils.randomAlphabetic(10))); } } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/audit/AuditIntegrationTest.java ================================================ package com.provectus.kafka.ui.service.audit; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.json.JsonMapper; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.TopicCreationDTO; import com.provectus.kafka.ui.model.rbac.Resource; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.BytesDeserializer; import org.apache.kafka.common.serialization.StringDeserializer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.reactive.server.WebTestClient; import org.testcontainers.shaded.org.awaitility.Awaitility; public class AuditIntegrationTest extends AbstractIntegrationTest { @Autowired private WebTestClient webTestClient; @Test void auditRecordWrittenIntoKafkaWhenNewTopicCreated() { String newTopicName = "test_audit_" + UUID.randomUUID(); webTestClient.post() .uri("/api/clusters/{clusterName}/topics", LOCAL) .bodyValue( new TopicCreationDTO() .replicationFactor(1) .partitions(1) .name(newTopicName) ) .exchange() .expectStatus() .isOk(); try (var consumer = createConsumer()) { var jsonMapper = new JsonMapper(); consumer.subscribe(List.of("__kui-audit-log")); Awaitility.await() .pollInSameThread() .atMost(Duration.ofSeconds(15)) .untilAsserted(() -> { var polled = consumer.poll(Duration.ofSeconds(1)); assertThat(polled).anySatisfy(kafkaRecord -> { try { AuditRecord record = jsonMapper.readValue(kafkaRecord.value(), AuditRecord.class); assertThat(record.operation()).isEqualTo("createTopic"); assertThat(record.resources()).map(AuditRecord.AuditResource::type).contains(Resource.TOPIC); assertThat(record.result().success()).isTrue(); assertThat(record.timestamp()).isNotBlank(); assertThat(record.clusterName()).isEqualTo(LOCAL); assertThat(record.operationParams()) .isEqualTo(Map.of( "name", newTopicName, "partitions", 1, "replicationFactor", 1, "configs", Map.of() )); } catch (JsonProcessingException e) { Assertions.fail(); } }); }); } } private KafkaConsumer createConsumer() { Properties props = new Properties(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(ConsumerConfig.GROUP_ID_CONFIG, AuditIntegrationTest.class.getName()); return new KafkaConsumer<>(props); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/audit/AuditServiceTest.java ================================================ package com.provectus.kafka.ui.service.audit; import static com.provectus.kafka.ui.service.audit.AuditService.createAuditWriter; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.service.ReactiveAdminClient; import java.util.Map; import java.util.Set; import java.util.function.Supplier; import org.apache.kafka.clients.producer.KafkaProducer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; import reactor.core.publisher.Signal; class AuditServiceTest { @Test void isAuditTopicChecksIfAuditIsEnabledForCluster() { Map writers = Map.of( "c1", new AuditWriter("с1", true, "c1topic", null, null), "c2", new AuditWriter("c2", false, "c2topic", mock(KafkaProducer.class), null) ); var auditService = new AuditService(writers); assertThat(auditService.isAuditTopic(KafkaCluster.builder().name("notExist").build(), "some")) .isFalse(); assertThat(auditService.isAuditTopic(KafkaCluster.builder().name("c1").build(), "c1topic")) .isFalse(); assertThat(auditService.isAuditTopic(KafkaCluster.builder().name("c2").build(), "c2topic")) .isTrue(); } @Test void auditCallsWriterMethodDependingOnSignal() { var auditWriter = mock(AuditWriter.class); var auditService = new AuditService(Map.of("test", auditWriter)); var cxt = AccessContext.builder().cluster("test").build(); auditService.audit(cxt, Signal.complete()); verify(auditWriter).write(any(), any(), eq(null)); var th = new Exception("testError"); auditService.audit(cxt, Signal.error(th)); verify(auditWriter).write(any(), any(), eq(th)); } @Nested class CreateAuditWriter { private final ReactiveAdminClient adminClientMock = mock(ReactiveAdminClient.class); private final Supplier> producerSupplierMock = mock(Supplier.class); private final ClustersProperties.Cluster clustersProperties = new ClustersProperties.Cluster(); private final KafkaCluster cluster = KafkaCluster .builder() .name("test") .originalProperties(clustersProperties) .build(); @BeforeEach void init() { when(producerSupplierMock.get()) .thenReturn(mock(KafkaProducer.class)); } @Test void logOnlyAlterOpsByDefault() { var auditProps = new ClustersProperties.AuditProperties(); auditProps.setConsoleAuditEnabled(true); clustersProperties.setAudit(auditProps); var maybeWriter = createAuditWriter(cluster, () -> adminClientMock, producerSupplierMock); assertThat(maybeWriter) .hasValueSatisfying(w -> assertThat(w.logAlterOperationsOnly()).isTrue()); } @Test void noWriterIfNoAuditPropsSet() { var maybeWriter = createAuditWriter(cluster, () -> adminClientMock, producerSupplierMock); assertThat(maybeWriter).isEmpty(); } @Test void setsLoggerIfConsoleLoggingEnabled() { var auditProps = new ClustersProperties.AuditProperties(); auditProps.setConsoleAuditEnabled(true); clustersProperties.setAudit(auditProps); var maybeWriter = createAuditWriter(cluster, () -> adminClientMock, producerSupplierMock); assertThat(maybeWriter).isPresent(); var writer = maybeWriter.get(); assertThat(writer.consoleLogger()).isNotNull(); } @Nested class WhenTopicAuditEnabled { @BeforeEach void setTopicWriteProperties() { var auditProps = new ClustersProperties.AuditProperties(); auditProps.setTopicAuditEnabled(true); auditProps.setTopic("test_audit_topic"); auditProps.setAuditTopicsPartitions(3); auditProps.setAuditTopicProperties(Map.of("p1", "v1")); clustersProperties.setAudit(auditProps); } @Test void createsProducerIfTopicExists() { when(adminClientMock.listTopics(true)) .thenReturn(Mono.just(Set.of("test_audit_topic"))); var maybeWriter = createAuditWriter(cluster, () -> adminClientMock, producerSupplierMock); assertThat(maybeWriter).isPresent(); //checking there was no topic creation request verify(adminClientMock, times(0)) .createTopic(any(), anyInt(), anyInt(), anyMap()); var writer = maybeWriter.get(); assertThat(writer.producer()).isNotNull(); assertThat(writer.targetTopic()).isEqualTo("test_audit_topic"); } @Test void createsProducerAndTopicIfItIsNotExist() { when(adminClientMock.listTopics(true)) .thenReturn(Mono.just(Set.of())); when(adminClientMock.createTopic(eq("test_audit_topic"), eq(3), eq(null), anyMap())) .thenReturn(Mono.empty()); var maybeWriter = createAuditWriter(cluster, () -> adminClientMock, producerSupplierMock); assertThat(maybeWriter).isPresent(); //verifying topic created verify(adminClientMock).createTopic(eq("test_audit_topic"), eq(3), eq(null), anyMap()); var writer = maybeWriter.get(); assertThat(writer.producer()).isNotNull(); assertThat(writer.targetTopic()).isEqualTo("test_audit_topic"); } } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/audit/AuditWriterTest.java ================================================ package com.provectus.kafka.ui.service.audit; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import com.provectus.kafka.ui.config.auth.AuthenticatedUser; import com.provectus.kafka.ui.model.rbac.AccessContext; import com.provectus.kafka.ui.model.rbac.AccessContext.AccessContextBuilder; import com.provectus.kafka.ui.model.rbac.permission.AclAction; import com.provectus.kafka.ui.model.rbac.permission.ClusterConfigAction; import com.provectus.kafka.ui.model.rbac.permission.ConnectAction; import com.provectus.kafka.ui.model.rbac.permission.ConsumerGroupAction; import com.provectus.kafka.ui.model.rbac.permission.SchemaAction; import com.provectus.kafka.ui.model.rbac.permission.TopicAction; import java.util.List; import java.util.function.UnaryOperator; import java.util.stream.Stream; import org.apache.kafka.clients.producer.KafkaProducer; import org.junit.jupiter.api.Nested; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; import org.slf4j.Logger; class AuditWriterTest { final KafkaProducer producerMock = Mockito.mock(KafkaProducer.class); final Logger loggerMock = Mockito.mock(Logger.class); final AuthenticatedUser user = new AuthenticatedUser("someone", List.of()); @Nested class AlterOperationsOnlyWriter { final AuditWriter alterOnlyWriter = new AuditWriter("test", true, "test-topic", producerMock, loggerMock); @ParameterizedTest @MethodSource void onlyLogsWhenAlterOperationIsPresentForOneOfResources(AccessContext ctxWithAlterOperation) { alterOnlyWriter.write(ctxWithAlterOperation, user, null); verify(producerMock).send(any(), any()); verify(loggerMock).info(any()); } static Stream onlyLogsWhenAlterOperationIsPresentForOneOfResources() { Stream> topicEditActions = TopicAction.ALTER_ACTIONS.stream().map(a -> c -> c.topic("test").topicActions(a)); Stream> clusterConfigEditActions = ClusterConfigAction.ALTER_ACTIONS.stream().map(a -> c -> c.clusterConfigActions(a)); Stream> aclEditActions = AclAction.ALTER_ACTIONS.stream().map(a -> c -> c.aclActions(a)); Stream> cgEditActions = ConsumerGroupAction.ALTER_ACTIONS.stream().map(a -> c -> c.consumerGroup("cg").consumerGroupActions(a)); Stream> schemaEditActions = SchemaAction.ALTER_ACTIONS.stream().map(a -> c -> c.schema("sc").schemaActions(a)); Stream> connEditActions = ConnectAction.ALTER_ACTIONS.stream().map(a -> c -> c.connect("conn").connectActions(a)); return Stream.of( topicEditActions, clusterConfigEditActions, aclEditActions, cgEditActions, connEditActions, schemaEditActions ) .flatMap(c -> c) .map(setter -> setter.apply(AccessContext.builder().cluster("test").operationName("test")).build()); } @ParameterizedTest @MethodSource void doesNothingIfNoResourceHasAlterAction(AccessContext readOnlyCxt) { alterOnlyWriter.write(readOnlyCxt, user, null); verifyNoInteractions(producerMock); verifyNoInteractions(loggerMock); } static Stream doesNothingIfNoResourceHasAlterAction() { return Stream.>of( c -> c.topic("test").topicActions(TopicAction.VIEW), c -> c.clusterConfigActions(ClusterConfigAction.VIEW), c -> c.aclActions(AclAction.VIEW), c -> c.consumerGroup("cg").consumerGroupActions(ConsumerGroupAction.VIEW), c -> c.schema("sc").schemaActions(SchemaAction.VIEW), c -> c.connect("conn").connectActions(ConnectAction.VIEW) ).map(setter -> setter.apply(AccessContext.builder().cluster("test").operationName("test")).build()); } } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/ConnectorsExporterTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.connect.model.ConnectorTopics; import com.provectus.kafka.ui.model.ConnectDTO; import com.provectus.kafka.ui.model.ConnectorDTO; import com.provectus.kafka.ui.model.ConnectorTypeDTO; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.service.KafkaConnectService; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import org.opendatadiscovery.client.model.DataEntity; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; class ConnectorsExporterTest { private static final KafkaCluster CLUSTER = KafkaCluster.builder() .name("test cluster") .bootstrapServers("localhost:9092") .build(); private final KafkaConnectService kafkaConnectService = mock(KafkaConnectService.class); private final ConnectorsExporter exporter = new ConnectorsExporter(kafkaConnectService); @Test void exportsConnectorsAsDataTransformers() { ConnectDTO connect = new ConnectDTO(); connect.setName("testConnect"); connect.setAddress("http://kconnect:8083"); ConnectorDTO sinkConnector = new ConnectorDTO(); sinkConnector.setName("testSink"); sinkConnector.setType(ConnectorTypeDTO.SINK); sinkConnector.setConnect(connect.getName()); sinkConnector.setConfig( Map.of( "connector.class", "FileStreamSink", "file", "filePathHere", "topic", "inputTopic" ) ); ConnectorDTO sourceConnector = new ConnectorDTO(); sourceConnector.setName("testSource"); sourceConnector.setConnect(connect.getName()); sourceConnector.setType(ConnectorTypeDTO.SOURCE); sourceConnector.setConfig( Map.of( "connector.class", "FileStreamSource", "file", "filePathHere", "topic", "outputTopic" ) ); when(kafkaConnectService.getConnects(CLUSTER)) .thenReturn(Flux.just(connect)); when(kafkaConnectService.getConnectorNamesWithErrorsSuppress(CLUSTER, connect.getName())) .thenReturn(Flux.just(sinkConnector.getName(), sourceConnector.getName())); when(kafkaConnectService.getConnector(CLUSTER, connect.getName(), sinkConnector.getName())) .thenReturn(Mono.just(sinkConnector)); when(kafkaConnectService.getConnector(CLUSTER, connect.getName(), sourceConnector.getName())) .thenReturn(Mono.just(sourceConnector)); when(kafkaConnectService.getConnectorTopics(CLUSTER, connect.getName(), sourceConnector.getName())) .thenReturn(Mono.just(new ConnectorTopics().topics(List.of("outputTopic")))); when(kafkaConnectService.getConnectorTopics(CLUSTER, connect.getName(), sinkConnector.getName())) .thenReturn(Mono.just(new ConnectorTopics().topics(List.of("inputTopic")))); StepVerifier.create(exporter.export(CLUSTER)) .assertNext(dataEntityList -> { assertThat(dataEntityList.getDataSourceOddrn()) .isEqualTo("//kafkaconnect/host/kconnect:8083"); assertThat(dataEntityList.getItems()) .hasSize(2); assertThat(dataEntityList.getItems()) .filteredOn(DataEntity::getOddrn, "//kafkaconnect/host/kconnect:8083/connectors/testSink") .singleElement() .satisfies(sink -> { assertThat(sink.getMetadata().get(0).getMetadata()) .containsOnlyKeys("type", "connector.class", "file", "topic"); assertThat(sink.getDataTransformer().getInputs()).contains( "//kafka/cluster/localhost:9092/topics/inputTopic"); }); assertThat(dataEntityList.getItems()) .filteredOn(DataEntity::getOddrn, "//kafkaconnect/host/kconnect:8083/connectors/testSource") .singleElement() .satisfies(source -> { assertThat(source.getMetadata().get(0).getMetadata()) .containsOnlyKeys("type", "connector.class", "file", "topic"); assertThat(source.getDataTransformer().getOutputs()).contains( "//kafka/cluster/localhost:9092/topics/outputTopic"); }); }) .verifyComplete(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/SchemaReferencesResolverTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.sr.model.SchemaReference; import com.provectus.kafka.ui.sr.model.SchemaSubject; import java.util.List; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; class SchemaReferencesResolverTest { private final KafkaSrClientApi srClientMock = mock(KafkaSrClientApi.class); private final SchemaReferencesResolver schemaReferencesResolver = new SchemaReferencesResolver(srClientMock); @Test void resolvesRefsUsingSrClient() { mockSrCall("sub1", 1, new SchemaSubject() .schema("schema1")); mockSrCall("sub2", 1, new SchemaSubject() .schema("schema2") .references( List.of( new SchemaReference().name("ref2_1").subject("sub2_1").version(2), new SchemaReference().name("ref2_2").subject("sub1").version(1)))); mockSrCall("sub2_1", 2, new SchemaSubject() .schema("schema2_1") .references( List.of( new SchemaReference().name("ref2_1_1").subject("sub2_1_1").version(3), new SchemaReference().name("ref1").subject("should_not_be_called").version(1) )) ); mockSrCall("sub2_1_1", 3, new SchemaSubject() .schema("schema2_1_1")); var resolvedRefsMono = schemaReferencesResolver.resolve( List.of( new SchemaReference().name("ref1").subject("sub1").version(1), new SchemaReference().name("ref2").subject("sub2").version(1))); StepVerifier.create(resolvedRefsMono) .assertNext(refs -> assertThat(refs) .containsExactlyEntriesOf( // checking map should be ordered ImmutableMap.builder() .put("ref1", "schema1") .put("ref2_1_1", "schema2_1_1") .put("ref2_1", "schema2_1") .put("ref2_2", "schema1") .put("ref2", "schema2") .build())) .verifyComplete(); } @Test void returnsEmptyMapOnEmptyInputs() { StepVerifier.create(schemaReferencesResolver.resolve(null)) .assertNext(map -> assertThat(map).isEmpty()) .verifyComplete(); StepVerifier.create(schemaReferencesResolver.resolve(List.of())) .assertNext(map -> assertThat(map).isEmpty()) .verifyComplete(); } private void mockSrCall(String subject, int version, SchemaSubject subjectToReturn) { when(srClientMock.getSubjectVersion(subject, version + "", true)) .thenReturn(Mono.just(subjectToReturn)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/TopicsExporterTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.Statistics; import com.provectus.kafka.ui.service.StatisticsCache; import com.provectus.kafka.ui.sr.api.KafkaSrClientApi; import com.provectus.kafka.ui.sr.model.SchemaSubject; import com.provectus.kafka.ui.sr.model.SchemaType; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.List; import java.util.Map; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionInfo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.opendatadiscovery.client.model.DataEntity; import org.opendatadiscovery.client.model.DataEntityType; import org.springframework.http.HttpHeaders; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; class TopicsExporterTest { private final KafkaSrClientApi schemaRegistryClientMock = mock(KafkaSrClientApi.class); private final KafkaCluster cluster = KafkaCluster.builder() .name("testCluster") .bootstrapServers("localhost:9092,localhost:19092") .schemaRegistryClient(ReactiveFailover.createNoop(schemaRegistryClientMock)) .build(); private Statistics stats; private TopicsExporter topicsExporter; @BeforeEach void init() { var statisticsCacheMock = mock(StatisticsCache.class); when(statisticsCacheMock.get(cluster)).thenAnswer(invocationOnMock -> stats); topicsExporter = new TopicsExporter( topic -> !topic.startsWith("_"), statisticsCacheMock ); } @Test void doesNotExportTopicsWhichDontFitFiltrationRule() { when(schemaRegistryClientMock.getSubjectVersion(anyString(), anyString(), anyBoolean())) .thenReturn(Mono.error(WebClientResponseException.create(404, "NF", new HttpHeaders(), null, null, null))); stats = Statistics.empty() .toBuilder() .topicDescriptions( Map.of( "_hidden", new TopicDescription("_hidden", false, List.of( new TopicPartitionInfo(0, null, List.of(), List.of()) )), "visible", new TopicDescription("visible", false, List.of( new TopicPartitionInfo(0, null, List.of(), List.of()) )) ) ) .build(); StepVerifier.create(topicsExporter.export(cluster)) .assertNext(entityList -> { assertThat(entityList.getDataSourceOddrn()) .isNotEmpty(); assertThat(entityList.getItems()) .hasSize(1) .allSatisfy(e -> e.getOddrn().contains("visible")); }) .verifyComplete(); } @Test void doesExportTopicData() { when(schemaRegistryClientMock.getSubjectVersion("testTopic-value", "latest", false)) .thenReturn(Mono.just( new SchemaSubject() .schema("\"string\"") .schemaType(SchemaType.AVRO) )); when(schemaRegistryClientMock.getSubjectVersion("testTopic-key", "latest", false)) .thenReturn(Mono.just( new SchemaSubject() .schema("\"int\"") .schemaType(SchemaType.AVRO) )); stats = Statistics.empty() .toBuilder() .topicDescriptions( Map.of( "testTopic", new TopicDescription( "testTopic", false, List.of( new TopicPartitionInfo( 0, null, List.of( new Node(1, "host1", 9092), new Node(2, "host2", 9092) ), List.of()) )) ) ) .topicConfigs( Map.of( "testTopic", List.of( new ConfigEntry( "custom.config", "100500", ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG, false, false, List.of(), ConfigEntry.ConfigType.INT, null ) ) ) ) .build(); StepVerifier.create(topicsExporter.export(cluster)) .assertNext(entityList -> { assertThat(entityList.getItems()) .hasSize(1); DataEntity topicEntity = entityList.getItems().get(0); assertThat(topicEntity.getName()).isNotEmpty(); assertThat(topicEntity.getOddrn()) .isEqualTo("//kafka/cluster/localhost:19092,localhost:9092/topics/testTopic"); assertThat(topicEntity.getType()).isEqualTo(DataEntityType.KAFKA_TOPIC); assertThat(topicEntity.getMetadata()) .hasSize(1) .singleElement() .satisfies(e -> assertThat(e.getMetadata()) .containsExactlyInAnyOrderEntriesOf( Map.of( "partitions", 1, "replication_factor", 2, "custom.config", "100500"))); assertThat(topicEntity.getDataset()).isNotNull(); assertThat(topicEntity.getDataset().getFieldList()) .hasSize(4); // 2 field for key, 2 for value }) .verifyComplete(); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/schema/AvroExtractorTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import static org.assertj.core.api.Assertions.assertThat; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.oddrn.model.KafkaPath; class AvroExtractorTest { @ParameterizedTest @ValueSource(booleans = {true, false}) void test(boolean isKey) { var list = AvroExtractor.extract( new AvroSchema(""" { "type": "record", "name": "Message", "namespace": "com.provectus.kafka", "fields": [ { "name": "f1", "type": { "type": "array", "items": { "type": "record", "name": "ArrElement", "fields": [ { "name": "longmap", "type": { "type": "map", "values": "long" } } ] } } }, { "name": "f2", "type": { "type": "record", "name": "InnerMessage", "fields": [ { "name": "text", "doc": "string field here", "type": "string" }, { "name": "innerMsgRef", "type": "InnerMessage" }, { "name": "nullable_union", "type": [ "null", "string", "int" ], "default": null }, { "name": "order_enum", "type": { "type": "enum", "name": "Suit", "symbols": [ "SPADES", "HEARTS" ] } }, { "name": "str_list", "type": { "type": "array", "items": "string" } } ] } } ] } """), KafkaPath.builder() .cluster("localhost:9092") .topic("someTopic") .build(), isKey ); String baseOddrn = "//kafka/cluster/localhost:9092/topics/someTopic/columns/" + (isKey ? "key" : "value"); assertThat(list).contains( DataSetFieldsExtractors.rootField( KafkaPath.builder().cluster("localhost:9092").topic("someTopic").build(), isKey ), new DataSetField() .name("f1") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/f1") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.LIST) .logicalType("array") .isNullable(false) ), new DataSetField() .name("ArrElement") .parentFieldOddrn(baseOddrn + "/f1") .oddrn(baseOddrn + "/f1/items/ArrElement") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("com.provectus.kafka.ArrElement") .isNullable(false) ), new DataSetField() .name("longmap") .parentFieldOddrn(baseOddrn + "/f1/items/ArrElement") .oddrn(baseOddrn + "/f1/items/ArrElement/fields/longmap") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.MAP) .logicalType("map") .isNullable(false) ), new DataSetField() .name("key") .parentFieldOddrn(baseOddrn + "/f1/items/ArrElement/fields/longmap") .oddrn(baseOddrn + "/f1/items/ArrElement/fields/longmap/key") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("string") .isNullable(false) ), new DataSetField() .name("value") .parentFieldOddrn(baseOddrn + "/f1/items/ArrElement/fields/longmap") .oddrn(baseOddrn + "/f1/items/ArrElement/fields/longmap/value") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("long") .isNullable(false) ), new DataSetField() .name("f2") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/f2") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("com.provectus.kafka.InnerMessage") .isNullable(false) ), new DataSetField() .name("text") .parentFieldOddrn(baseOddrn + "/f2") .oddrn(baseOddrn + "/f2/fields/text") .description("string field here") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("string") .isNullable(false) ), new DataSetField() .name("innerMsgRef") .parentFieldOddrn(baseOddrn + "/f2") .oddrn(baseOddrn + "/f2/fields/innerMsgRef") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("com.provectus.kafka.InnerMessage") .isNullable(false) ), new DataSetField() .name("nullable_union") .parentFieldOddrn(baseOddrn + "/f2") .oddrn(baseOddrn + "/f2/fields/nullable_union") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.UNION) .logicalType("union") .isNullable(true) ), new DataSetField() .name("string") .parentFieldOddrn(baseOddrn + "/f2/fields/nullable_union") .oddrn(baseOddrn + "/f2/fields/nullable_union/values/string") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("string") .isNullable(true) ), new DataSetField() .name("int") .parentFieldOddrn(baseOddrn + "/f2/fields/nullable_union") .oddrn(baseOddrn + "/f2/fields/nullable_union/values/int") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("int") .isNullable(true) ), new DataSetField() .name("int") .parentFieldOddrn(baseOddrn + "/f2/fields/nullable_union") .oddrn(baseOddrn + "/f2/fields/nullable_union/values/int") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("int") .isNullable(true) ), new DataSetField() .name("order_enum") .parentFieldOddrn(baseOddrn + "/f2") .oddrn(baseOddrn + "/f2/fields/order_enum") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("enum") .isNullable(false) ), new DataSetField() .name("str_list") .parentFieldOddrn(baseOddrn + "/f2") .oddrn(baseOddrn + "/f2/fields/str_list") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.LIST) .logicalType("array") .isNullable(false) ), new DataSetField() .name("string") .parentFieldOddrn(baseOddrn + "/f2/fields/str_list") .oddrn(baseOddrn + "/f2/fields/str_list/items/string") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("string") .isNullable(false) ) ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/schema/JsonSchemaExtractorTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import static org.assertj.core.api.Assertions.assertThat; import io.confluent.kafka.schemaregistry.json.JsonSchema; import java.net.URI; import java.util.List; import java.util.Map; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.client.model.MetadataExtension; import org.opendatadiscovery.oddrn.model.KafkaPath; class JsonSchemaExtractorTest { @ParameterizedTest @ValueSource(booleans = {true, false}) void test(boolean isKey) { String jsonSchema = """ { "$id": "http://example.com/test.TestMsg", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "required": [ "int32_field" ], "properties": { "int32_field": { "type": "integer", "title": "field title" }, "lst_s_field": { "type": "array", "items": { "type": "string" }, "description": "field descr" }, "untyped_struct_field": { "type": "object", "properties": {} }, "union_field": { "type": [ "number", "object", "null" ] }, "struct_field": { "type": "object", "properties": { "bool_field": { "type": "boolean" } } } } } """; var fields = JsonSchemaExtractor.extract( new JsonSchema(jsonSchema), KafkaPath.builder() .cluster("localhost:9092") .topic("someTopic") .build(), isKey ); String baseOddrn = "//kafka/cluster/localhost:9092/topics/someTopic/columns/" + (isKey ? "key" : "value"); assertThat(fields).contains( DataSetFieldsExtractors.rootField( KafkaPath.builder().cluster("localhost:9092").topic("someTopic").build(), isKey ), new DataSetField() .name("int32_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/int32_field") .description("field title") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.NUMBER) .logicalType("Number") .isNullable(false)), new DataSetField() .name("lst_s_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/lst_s_field") .description("field descr") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.LIST) .logicalType("array") .isNullable(true)), new DataSetField() .name("String") .parentFieldOddrn(baseOddrn + "/lst_s_field") .oddrn(baseOddrn + "/lst_s_field/items/String") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("String") .isNullable(false)), new DataSetField() .name("untyped_struct_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/untyped_struct_field") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("Object") .isNullable(true)), new DataSetField() .name("union_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/union_field/anyOf") .metadata(List.of(new MetadataExtension() .schemaUrl(URI.create("wontbeused.oops")) .metadata(Map.of("criterion", "anyOf")))) .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.UNION) .logicalType("anyOf") .isNullable(true)), new DataSetField() .name("Number") .parentFieldOddrn(baseOddrn + "/union_field/anyOf") .oddrn(baseOddrn + "/union_field/anyOf/values/Number") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.NUMBER) .logicalType("Number") .isNullable(true)), new DataSetField() .name("Object") .parentFieldOddrn(baseOddrn + "/union_field/anyOf") .oddrn(baseOddrn + "/union_field/anyOf/values/Object") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("Object") .isNullable(true)), new DataSetField() .name("Null") .parentFieldOddrn(baseOddrn + "/union_field/anyOf") .oddrn(baseOddrn + "/union_field/anyOf/values/Null") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.UNKNOWN) .logicalType("Null") .isNullable(true)), new DataSetField() .name("struct_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/struct_field") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("Object") .isNullable(true)), new DataSetField() .name("bool_field") .parentFieldOddrn(baseOddrn + "/struct_field") .oddrn(baseOddrn + "/struct_field/fields/bool_field") .type(new DataSetFieldType() .type(DataSetFieldType.TypeEnum.BOOLEAN) .logicalType("Boolean") .isNullable(true)) ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/integration/odd/schema/ProtoExtractorTest.java ================================================ package com.provectus.kafka.ui.service.integration.odd.schema; import static org.assertj.core.api.Assertions.assertThat; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.opendatadiscovery.client.model.DataSetField; import org.opendatadiscovery.client.model.DataSetFieldType; import org.opendatadiscovery.oddrn.model.KafkaPath; class ProtoExtractorTest { @ParameterizedTest @ValueSource(booleans = {true, false}) void test(boolean isKey) { String protoSchema = """ syntax = "proto3"; package test; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; message TestMsg { map mapField = 100; int32 int32_field = 2; bool bool_field = 3; SampleEnum enum_field = 4; enum SampleEnum { ENUM_V1 = 0; ENUM_V2 = 1; } google.protobuf.Timestamp ts_field = 5; google.protobuf.Duration duration_field = 8; oneof some_oneof1 { google.protobuf.Value one_of_v1 = 9; google.protobuf.Value one_of_v2 = 10; } // wrapper field: google.protobuf.Int64Value int64_w_field = 11; //embedded msg EmbeddedMsg emb = 19; message EmbeddedMsg { int32 emb_f1 = 1; TestMsg outer_ref = 2; } }"""; var list = ProtoExtractor.extract( new ProtobufSchema(protoSchema), KafkaPath.builder() .cluster("localhost:9092") .topic("someTopic") .build(), isKey ); String baseOddrn = "//kafka/cluster/localhost:9092/topics/someTopic/columns/" + (isKey ? "key" : "value"); assertThat(list) .contains( DataSetFieldsExtractors.rootField( KafkaPath.builder().cluster("localhost:9092").topic("someTopic").build(), isKey ), new DataSetField() .name("mapField") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/mapField") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.LIST) .logicalType("repeated") .isNullable(true) ), new DataSetField() .name("int32_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/int32_field") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("int32") .isNullable(true) ), new DataSetField() .name("enum_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/enum_field") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRING) .logicalType("enum") .isNullable(true) ), new DataSetField() .name("ts_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/ts_field") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.DATETIME) .logicalType("google.protobuf.Timestamp") .isNullable(true) ), new DataSetField() .name("duration_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/duration_field") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.DURATION) .logicalType("google.protobuf.Duration") .isNullable(true) ), new DataSetField() .name("one_of_v1") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/one_of_v1") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.UNKNOWN) .logicalType("google.protobuf.Value") .isNullable(true) ), new DataSetField() .name("one_of_v2") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/one_of_v2") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.UNKNOWN) .logicalType("google.protobuf.Value") .isNullable(true) ), new DataSetField() .name("int64_w_field") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/int64_w_field") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("google.protobuf.Int64Value") .isNullable(true) ), new DataSetField() .name("emb") .parentFieldOddrn(baseOddrn) .oddrn(baseOddrn + "/emb") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("test.TestMsg.EmbeddedMsg") .isNullable(true) ), new DataSetField() .name("emb_f1") .parentFieldOddrn(baseOddrn + "/emb") .oddrn(baseOddrn + "/emb/fields/emb_f1") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.INTEGER) .logicalType("int32") .isNullable(true) ), new DataSetField() .name("outer_ref") .parentFieldOddrn(baseOddrn + "/emb") .oddrn(baseOddrn + "/emb/fields/outer_ref") .type( new DataSetFieldType() .type(DataSetFieldType.TypeEnum.STRUCT) .logicalType("test.TestMsg") .isNullable(true) ) ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlApiClientTest.java ================================================ package com.provectus.kafka.ui.service.ksql; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.TextNode; import com.provectus.kafka.ui.AbstractIntegrationTest; import java.math.BigDecimal; import java.time.Duration; import java.util.Map; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.shaded.org.awaitility.Awaitility; import reactor.test.StepVerifier; class KsqlApiClientTest extends AbstractIntegrationTest { @BeforeAll static void startContainer() { KSQL_DB.start(); } @AfterAll static void stopContainer() { KSQL_DB.stop(); } // Tutorial is here: https://ksqldb.io/quickstart.html @Test void ksqTutorialQueriesWork() { var client = ksqlClient(); execCommandSync(client, "CREATE STREAM riderLocations (profileId VARCHAR, latitude DOUBLE, longitude DOUBLE) " + "WITH (kafka_topic='locations', value_format='json', partitions=1);", "CREATE TABLE currentLocation AS " + " SELECT profileId, " + " LATEST_BY_OFFSET(latitude) AS la, " + " LATEST_BY_OFFSET(longitude) AS lo " + " FROM riderlocations " + " GROUP BY profileId " + " EMIT CHANGES;", "CREATE TABLE ridersNearMountainView AS " + " SELECT ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1) AS distanceInMiles, " + " COLLECT_LIST(profileId) AS riders, " + " COUNT(*) AS count " + " FROM currentLocation " + " GROUP BY ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1);", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('c2309eec', 37.7877, -122.4205); ", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('18f4ea86', 37.3903, -122.0643); ", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4ab5cbad', 37.3952, -122.0813); ", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('8b6eae59', 37.3944, -122.0813); ", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4a7c7b41', 37.4049, -122.0822); ", "INSERT INTO riderLocations (profileId, latitude, longitude) VALUES ('4ddad000', 37.7857, -122.4011);" ); Awaitility.await() .pollDelay(Duration.ofSeconds(1)) .atMost(Duration.ofSeconds(20)) .untilAsserted(() -> assertLastKsqTutorialQueryResult(client)); } private void assertLastKsqTutorialQueryResult(KsqlApiClient client) { // expected results: //{"header":"Schema","columnNames":[...],"values":null} //{"header":"Row","columnNames":null,"values":[[0,["4ab5cbad","8b6eae59","4a7c7b41"],3]]} //{"header":"Row","columnNames":null,"values":[[10.0,["18f4ea86"],1]]} StepVerifier.create( client.execute( "SELECT * from ridersNearMountainView WHERE distanceInMiles <= 10;", Map.of() ) ) .assertNext(header -> { assertThat(header.getHeader()).isEqualTo("Schema"); assertThat(header.getColumnNames()).hasSize(3); assertThat(header.getValues()).isNull(); }) .assertNext(row -> { var distance = (DecimalNode) row.getValues().get(0).get(0); var riders = (ArrayNode) row.getValues().get(0).get(1); var count = (IntNode) row.getValues().get(0).get(2); assertThat(distance).isEqualTo(new DecimalNode(new BigDecimal(0))); assertThat(riders).isEqualTo(new ArrayNode(JsonNodeFactory.instance) .add(new TextNode("4ab5cbad")) .add(new TextNode("8b6eae59")) .add(new TextNode("4a7c7b41"))); assertThat(count).isEqualTo(new IntNode(3)); }) .assertNext(row -> { var distance = (DecimalNode) row.getValues().get(0).get(0); var riders = (ArrayNode) row.getValues().get(0).get(1); var count = (IntNode) row.getValues().get(0).get(2); assertThat(distance).isEqualTo(new DecimalNode(new BigDecimal(10))); assertThat(riders).isEqualTo(new ArrayNode(JsonNodeFactory.instance) .add(new TextNode("18f4ea86"))); assertThat(count).isEqualTo(new IntNode(1)); }) .verifyComplete(); } private void execCommandSync(KsqlApiClient client, String... ksqls) { for (String ksql : ksqls) { client.execute(ksql, Map.of()).collectList().block(); } } private KsqlApiClient ksqlClient() { return new KsqlApiClient(KSQL_DB.url(), null, null, null, null); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java ================================================ package com.provectus.kafka.ui.service.ksql; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.AbstractIntegrationTest; import com.provectus.kafka.ui.model.KafkaCluster; import com.provectus.kafka.ui.model.KsqlStreamDescriptionDTO; import com.provectus.kafka.ui.model.KsqlTableDescriptionDTO; import com.provectus.kafka.ui.util.ReactiveFailover; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; class KsqlServiceV2Test extends AbstractIntegrationTest { private static final Set STREAMS_TO_DELETE = new CopyOnWriteArraySet<>(); private static final Set TABLES_TO_DELETE = new CopyOnWriteArraySet<>(); @BeforeAll static void init() { KSQL_DB.start(); } @AfterAll static void cleanup() { TABLES_TO_DELETE.forEach(t -> ksqlClient().execute(String.format("DROP TABLE IF EXISTS %s DELETE TOPIC;", t), Map.of()) .blockLast()); STREAMS_TO_DELETE.forEach(s -> ksqlClient().execute(String.format("DROP STREAM IF EXISTS %s DELETE TOPIC;", s), Map.of()) .blockLast()); KSQL_DB.stop(); } private final KsqlServiceV2 ksqlService = new KsqlServiceV2(); @Test void listStreamsReturnsAllKsqlStreams() { var streamName = "stream_" + System.currentTimeMillis(); STREAMS_TO_DELETE.add(streamName); ksqlClient() .execute( String.format("CREATE STREAM %s ( " + " c1 BIGINT KEY, " + " c2 VARCHAR " + " ) WITH ( " + " KAFKA_TOPIC = '%s_topic', " + " PARTITIONS = 1, " + " VALUE_FORMAT = 'JSON' " + " );", streamName, streamName), Map.of()) .blockLast(); var streams = ksqlService.listStreams(cluster()).collectList().block(); assertThat(streams).contains( new KsqlStreamDescriptionDTO() .name(streamName.toUpperCase()) .topic(streamName + "_topic") .keyFormat("KAFKA") .valueFormat("JSON") ); } @Test void listTablesReturnsAllKsqlTables() { var tableName = "table_" + System.currentTimeMillis(); TABLES_TO_DELETE.add(tableName); ksqlClient() .execute( String.format("CREATE TABLE %s ( " + " c1 BIGINT PRIMARY KEY, " + " c2 VARCHAR " + " ) WITH ( " + " KAFKA_TOPIC = '%s_topic', " + " PARTITIONS = 1, " + " VALUE_FORMAT = 'JSON' " + " );", tableName, tableName), Map.of()) .blockLast(); var tables = ksqlService.listTables(cluster()).collectList().block(); assertThat(tables).contains( new KsqlTableDescriptionDTO() .name(tableName.toUpperCase()) .topic(tableName + "_topic") .keyFormat("KAFKA") .valueFormat("JSON") .isWindowed(false) ); } private static KafkaCluster cluster() { return KafkaCluster.builder() .ksqlClient(ReactiveFailover.create( List.of(ksqlClient()), th -> true, "", ReactiveFailover.DEFAULT_RETRY_GRACE_PERIOD_MS)) .build(); } private static KsqlApiClient ksqlClient() { return new KsqlApiClient(KSQL_DB.url(), null, null, null, null); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/response/ResponseParserTest.java ================================================ package com.provectus.kafka.ui.service.ksql.response; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; class ResponseParserTest { @Test void parsesSelectHeaderIntoColumnNames() { assertThat(ResponseParser.parseSelectHeadersString("`inQuotes` INT, notInQuotes INT")) .containsExactly("`inQuotes` INT", "notInQuotes INT"); assertThat(ResponseParser.parseSelectHeadersString("`name with comma,` INT, name2 STRING")) .containsExactly("`name with comma,` INT", "name2 STRING"); assertThat(ResponseParser.parseSelectHeadersString( "`topLvl` INT, `struct` STRUCT<`nested1` STRING, anotherName STRUCT>")) .containsExactly( "`topLvl` INT", "`struct` STRUCT<`nested1` STRING, anotherName STRUCT>" ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/DataMaskingTest.java ================================================ package com.provectus.kafka.ui.service.masking; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.serde.api.Serde; import com.provectus.kafka.ui.service.masking.policies.MaskingPolicy; import java.util.List; import java.util.regex.Pattern; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; class DataMaskingTest { private static final String TOPIC = "test_topic"; private DataMasking masking; private MaskingPolicy policy1; private MaskingPolicy policy2; private MaskingPolicy policy3; @BeforeEach void init() { policy1 = spy(createMaskPolicy()); policy2 = spy(createMaskPolicy()); policy3 = spy(createMaskPolicy()); masking = new DataMasking( List.of( new DataMasking.Mask(Pattern.compile(TOPIC), null, policy1), new DataMasking.Mask(null, Pattern.compile(TOPIC), policy2), new DataMasking.Mask(null, Pattern.compile(TOPIC + "|otherTopic"), policy3))); } private MaskingPolicy createMaskPolicy() { var props = new ClustersProperties.Masking(); props.setType(ClustersProperties.Masking.Type.REMOVE); return MaskingPolicy.create(props); } @ParameterizedTest @ValueSource(strings = { "{\"some\": \"json\"}", "[ {\"json\": \"array\"} ]" }) @SneakyThrows void appliesMasksToJsonContainerArgsBasedOnTopicPatterns(String jsonObjOrArr) { var parsedJson = (ContainerNode) new JsonMapper().readTree(jsonObjOrArr); masking.getMaskingFunction(TOPIC, Serde.Target.KEY).apply(jsonObjOrArr); verify(policy1).applyToJsonContainer(eq(parsedJson)); verifyNoInteractions(policy2, policy3); reset(policy1, policy2, policy3); masking.getMaskingFunction(TOPIC, Serde.Target.VALUE).apply(jsonObjOrArr); verify(policy2).applyToJsonContainer(eq(parsedJson)); verify(policy3).applyToJsonContainer(eq(policy2.applyToJsonContainer(parsedJson))); verifyNoInteractions(policy1); } @ParameterizedTest @ValueSource(strings = { "non json str", "234", "null" }) void appliesFirstFoundMaskToStringArgsBasedOnTopicPatterns(String nonJsonObjOrArrString) { masking.getMaskingFunction(TOPIC, Serde.Target.KEY).apply(nonJsonObjOrArrString); verify(policy1).applyToString(eq(nonJsonObjOrArrString)); verifyNoInteractions(policy2, policy3); reset(policy1, policy2, policy3); masking.getMaskingFunction(TOPIC, Serde.Target.VALUE).apply(nonJsonObjOrArrString); verify(policy2).applyToString(eq(nonJsonObjOrArrString)); verifyNoInteractions(policy1, policy3); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/FieldsSelectorTest.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.provectus.kafka.ui.config.ClustersProperties; import com.provectus.kafka.ui.exception.ValidationException; import java.util.List; import org.junit.jupiter.api.Test; class FieldsSelectorTest { @Test void selectsFieldsDueToProvidedPattern() { var properties = new ClustersProperties.Masking(); properties.setFieldsNamePattern("f1|f2"); var selector = FieldsSelector.create(properties); assertThat(selector.shouldBeMasked("f1")).isTrue(); assertThat(selector.shouldBeMasked("f2")).isTrue(); assertThat(selector.shouldBeMasked("doesNotMatchPattern")).isFalse(); } @Test void selectsFieldsDueToProvidedFieldNames() { var properties = new ClustersProperties.Masking(); properties.setFields(List.of("f1", "f2")); var selector = FieldsSelector.create(properties); assertThat(selector.shouldBeMasked("f1")).isTrue(); assertThat(selector.shouldBeMasked("f2")).isTrue(); assertThat(selector.shouldBeMasked("notInAList")).isFalse(); } @Test void selectAllFieldsIfNoPatternAndNoNamesProvided() { var properties = new ClustersProperties.Masking(); var selector = FieldsSelector.create(properties); assertThat(selector.shouldBeMasked("anyPropertyName")).isTrue(); } @Test void throwsExceptionIfBothFieldListAndPatternProvided() { var properties = new ClustersProperties.Masking(); properties.setFieldsNamePattern("f1|f2"); properties.setFields(List.of("f3", "f4")); assertThatThrownBy(() -> FieldsSelector.create(properties)) .isInstanceOf(ValidationException.class); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/MaskTest.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import java.util.List; import java.util.stream.Stream; import lombok.SneakyThrows; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; class MaskTest { private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); private static final List PATTERN = List.of("X", "x", "n", "-"); @ParameterizedTest @MethodSource void testApplyToJsonContainer(FieldsSelector selector, ContainerNode original, ContainerNode expected) { Mask policy = new Mask(selector, PATTERN); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{ \"id\": \"nnn\", \"name\": { \"first\": \"Xxxxx\", \"surname\": \"Xxxxnnn-\"}}") ), Arguments.of( FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"id\": \"nnn\", \"f2\": 234}, { \"name\": \"n-n\", \"f2\": 345} ]") ), Arguments.of( FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Xxxxnnn-\"}}") ), Arguments.of( (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"Xxxxx\", \"name\": \"Xxxxnnn-\"}}") ) ); } @ParameterizedTest @CsvSource({ "Some string?!1, Xxxx xxxxxx--n", "1.24343, n-nnnnn", "null, xxxx" }) void testApplyToString(String original, String expected) { Mask policy = new Mask(fieldName -> true, PATTERN); assertThat(policy.applyToString(original)).isEqualTo(expected); } @SneakyThrows private static JsonNode parse(String str) { return new JsonMapper().readTree(str); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/RemoveTest.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import java.util.List; import java.util.stream.Stream; import lombok.SneakyThrows; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; class RemoveTest { private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); @ParameterizedTest @MethodSource void testApplyToJsonContainer(FieldsSelector fieldsSelector, ContainerNode original, ContainerNode expected) { var policy = new Remove(fieldsSelector); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{}") ), Arguments.of( FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"f2\": 234}, { \"f2\": 345} ]") ), Arguments.of( FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\"}}") ), Arguments.of( (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"v1\", \"f2\": \"v2\", \"inner\" : {\"if1\": \"iv1\"}}}"), parse("{}") ), Arguments.of( (FieldsSelector) (fieldName -> true), parse("[{ \"f1\": 123}, { \"f2\": \"1.2\"} ]"), parse("[{}, {}]") ) ); } @SneakyThrows private static JsonNode parse(String str) { return new JsonMapper().readTree(str); } @ParameterizedTest @CsvSource({ "Some string?!1, null", "1.24343, null", "null, null" }) void testApplyToString(String original, String expected) { var policy = new Remove(fieldName -> true); assertThat(policy.applyToString(original)).isEqualTo(expected); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/masking/policies/ReplaceTest.java ================================================ package com.provectus.kafka.ui.service.masking.policies; import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import java.util.List; import java.util.stream.Stream; import lombok.SneakyThrows; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; class ReplaceTest { private static final FieldsSelector FIELDS_SELECTOR = fieldName -> List.of("id", "name").contains(fieldName); private static final String REPLACEMENT_STRING = "***"; @ParameterizedTest @MethodSource void testApplyToJsonContainer(FieldsSelector fieldsSelector, ContainerNode original, ContainerNode expected) { var policy = new Replace(fieldsSelector, REPLACEMENT_STRING); assertThat(policy.applyToJsonContainer(original)).isEqualTo(expected); } private static Stream testApplyToJsonContainer() { return Stream.of( Arguments.of( FIELDS_SELECTOR, parse("{ \"id\": 123, \"name\": { \"first\": \"James\", \"surname\": \"Bond777!\"}}"), parse("{ \"id\": \"***\", \"name\": { \"first\": \"***\", \"surname\": \"***\"}}") ), Arguments.of( FIELDS_SELECTOR, parse("[{ \"id\": 123, \"f2\": 234}, { \"name\": \"1.2\", \"f2\": 345} ]"), parse("[{ \"id\": \"***\", \"f2\": 234}, { \"name\": \"***\", \"f2\": 345} ]") ), Arguments.of( FIELDS_SELECTOR, parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"Bond777!\"}}"), parse("{ \"outer\": { \"f1\": \"James\", \"name\": \"***\"}}") ), Arguments.of( (FieldsSelector) (fieldName -> true), parse("{ \"outer\": { \"f1\": \"v1\", \"f2\": \"v2\", \"inner\" : {\"if1\": \"iv1\"}}}"), parse("{ \"outer\": { \"f1\": \"***\", \"f2\": \"***\", \"inner\" : {\"if1\": \"***\"}}}}") ) ); } @SneakyThrows private static JsonNode parse(String str) { return new JsonMapper().readTree(str); } @ParameterizedTest @CsvSource({ "Some string?!1, ***", "1.24343, ***", "null, ***" }) void testApplyToString(String original, String expected) { var policy = new Replace(fieldName -> true, REPLACEMENT_STRING); assertThat(policy.applyToString(original)).isEqualTo(expected); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/JmxMetricsFormatterTest.java ================================================ package com.provectus.kafka.ui.service.metrics; import static org.assertj.core.api.Assertions.assertThat; import java.math.BigDecimal; import java.util.List; import java.util.Map; import javax.management.MBeanAttributeInfo; import javax.management.ObjectName; import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; class JmxMetricsFormatterTest { /** * Original format is here. */ @Test void convertsJmxMetricsAccordingToJmxExporterFormat() throws Exception { List metrics = JmxMetricsFormatter.constructMetricsList( new ObjectName( "kafka.server:type=Some.BrokerTopic-Metrics,name=BytesOutPer-Sec,topic=test,some-lbl=123"), new MBeanAttributeInfo[] { createMbeanInfo("FifteenMinuteRate"), createMbeanInfo("Mean"), createMbeanInfo("Calls-count"), createMbeanInfo("SkipValue"), }, new Object[] { 123.0, 100.0, 10L, "string values not supported" } ); assertThat(metrics).hasSize(3); assertMetricsEqual( RawMetric.create( "kafka_server_Some_BrokerTopic_Metrics_FifteenMinuteRate", Map.of("name", "BytesOutPer-Sec", "topic", "test", "some_lbl", "123"), BigDecimal.valueOf(123.0) ), metrics.get(0) ); assertMetricsEqual( RawMetric.create( "kafka_server_Some_BrokerTopic_Metrics_Mean", Map.of("name", "BytesOutPer-Sec", "topic", "test", "some_lbl", "123"), BigDecimal.valueOf(100.0) ), metrics.get(1) ); assertMetricsEqual( RawMetric.create( "kafka_server_Some_BrokerTopic_Metrics_Calls_count", Map.of("name", "BytesOutPer-Sec", "topic", "test", "some_lbl", "123"), BigDecimal.valueOf(10) ), metrics.get(2) ); } private static MBeanAttributeInfo createMbeanInfo(String name) { return new MBeanAttributeInfo(name, "sometype-notused", null, true, true, false, null); } private void assertMetricsEqual(RawMetric expected, RawMetric actual) { assertThat(actual.name()).isEqualTo(expected.name()); assertThat(actual.labels()).isEqualTo(expected.labels()); assertThat(actual.value()).isCloseTo(expected.value(), Offset.offset(new BigDecimal("0.001"))); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusEndpointMetricsParserTest.java ================================================ package com.provectus.kafka.ui.service.metrics; import static org.assertj.core.api.Assertions.assertThat; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Test; class PrometheusEndpointMetricsParserTest { @Test void test() { String metricsString = "kafka_server_BrokerTopicMetrics_FifteenMinuteRate" + "{name=\"BytesOutPerSec\",topic=\"__confluent.support.metrics\",} 123.1234"; Optional parsedOpt = PrometheusEndpointMetricsParser.parse(metricsString); assertThat(parsedOpt).hasValueSatisfying(metric -> { assertThat(metric.name()).isEqualTo("kafka_server_BrokerTopicMetrics_FifteenMinuteRate"); assertThat(metric.value()).isEqualTo("123.1234"); assertThat(metric.labels()).containsExactlyEntriesOf( Map.of( "name", "BytesOutPerSec", "topic", "__confluent.support.metrics" )); }); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/PrometheusMetricsRetrieverTest.java ================================================ package com.provectus.kafka.ui.service.metrics; import com.provectus.kafka.ui.model.MetricsConfig; import java.io.IOException; import java.math.BigDecimal; import java.util.List; import java.util.Map; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.web.reactive.function.client.WebClient; import reactor.test.StepVerifier; class PrometheusMetricsRetrieverTest { private final PrometheusMetricsRetriever retriever = new PrometheusMetricsRetriever(); private final MockWebServer mockWebServer = new MockWebServer(); @BeforeEach void startMockServer() throws IOException { mockWebServer.start(); } @AfterEach void stopMockServer() throws IOException { mockWebServer.close(); } @Test void callsMetricsEndpointAndConvertsResponceToRawMetric() { var url = mockWebServer.url("/metrics"); mockWebServer.enqueue(prepareResponse()); MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), null, null); StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) .expectNextSequence(expectedRawMetrics()) // third metric should not be present, since it has "NaN" value .verifyComplete(); } @Test void callsSecureMetricsEndpointAndConvertsResponceToRawMetric() { var url = mockWebServer.url("/metrics"); mockWebServer.enqueue(prepareResponse()); MetricsConfig metricsConfig = prepareMetricsConfig(url.port(), "username", "password"); StepVerifier.create(retriever.retrieve(WebClient.create(), url.host(), metricsConfig)) .expectNextSequence(expectedRawMetrics()) // third metric should not be present, since it has "NaN" value .verifyComplete(); } MockResponse prepareResponse() { // body copied from real jmx exporter return new MockResponse().setBody( "# HELP kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate Attribute exposed for management \n" + "# TYPE kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate untyped\n" + "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate{name=\"RequestHandlerAvgIdlePercent\",} 0.898\n" + "# HELP kafka_server_socket_server_metrics_request_size_avg The average size of requests sent. \n" + "# TYPE kafka_server_socket_server_metrics_request_size_avg untyped\n" + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN\",networkProcessor=\"1\",} 101.1\n" + "kafka_server_socket_server_metrics_request_size_avg{listener=\"PLAIN2\",networkProcessor=\"5\",} NaN" ); } MetricsConfig prepareMetricsConfig(Integer port, String username, String password) { return MetricsConfig.builder() .ssl(false) .port(port) .type(MetricsConfig.PROMETHEUS_METRICS_TYPE) .username(username) .password(password) .build(); } List expectedRawMetrics() { var firstMetric = RawMetric.create( "kafka_server_KafkaRequestHandlerPool_FifteenMinuteRate", Map.of("name", "RequestHandlerAvgIdlePercent"), new BigDecimal("0.898") ); var secondMetric = RawMetric.create( "kafka_server_socket_server_metrics_request_size_avg", Map.of("listener", "PLAIN", "networkProcessor", "1"), new BigDecimal("101.1") ); return List.of(firstMetric, secondMetric); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/metrics/WellKnownMetricsTest.java ================================================ package com.provectus.kafka.ui.service.metrics; import static org.assertj.core.api.Assertions.assertThat; import com.provectus.kafka.ui.model.Metrics; import java.math.BigDecimal; import java.util.Arrays; import java.util.Map; import java.util.Optional; import org.apache.kafka.common.Node; import org.junit.jupiter.api.Test; class WellKnownMetricsTest { private final WellKnownMetrics wellKnownMetrics = new WellKnownMetrics(); @Test void bytesIoTopicMetricsPopulated() { populateWith( new Node(0, "host", 123), "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",topic=\"test-topic\",} 1.0", "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",topic=\"test-topic\",} 2.0", "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", "kafka_server_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0", "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",topic=\"test-topic\",} 1.0", "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",topic=\"test-topic\",} 2.0" ); assertThat(wellKnownMetrics.bytesInFifteenMinuteRate) .containsEntry("test-topic", new BigDecimal("3.0")); assertThat(wellKnownMetrics.bytesOutFifteenMinuteRate) .containsEntry("test-topic", new BigDecimal("6.0")); } @Test void bytesIoBrokerMetricsPopulated() { populateWith( new Node(1, "host1", 123), "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesInPerSec\",} 1.0", "kafka_server_BrokerTopicMetrics_FifteenMinuteRate{name=\"BytesOutPerSec\",} 2.0" ); populateWith( new Node(2, "host2", 345), "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesinpersec\",} 10.0", "some_unknown_prefix_brokertopicmetrics_fifteenminuterate{name=\"bytesoutpersec\",} 20.0" ); assertThat(wellKnownMetrics.brokerBytesInFifteenMinuteRate) .hasSize(2) .containsEntry(1, new BigDecimal("1.0")) .containsEntry(2, new BigDecimal("10.0")); assertThat(wellKnownMetrics.brokerBytesOutFifteenMinuteRate) .hasSize(2) .containsEntry(1, new BigDecimal("2.0")) .containsEntry(2, new BigDecimal("20.0")); } @Test void appliesInnerStateToMetricsBuilder() { //filling per topic io rates wellKnownMetrics.bytesInFifteenMinuteRate.put("topic", new BigDecimal(1)); wellKnownMetrics.bytesOutFifteenMinuteRate.put("topic", new BigDecimal(2)); //filling per broker io rates wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(1, new BigDecimal(1)); wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(1, new BigDecimal(2)); wellKnownMetrics.brokerBytesInFifteenMinuteRate.put(2, new BigDecimal(10)); wellKnownMetrics.brokerBytesOutFifteenMinuteRate.put(2, new BigDecimal(20)); Metrics.MetricsBuilder builder = Metrics.builder(); wellKnownMetrics.apply(builder); var metrics = builder.build(); // checking per topic io rates assertThat(metrics.getTopicBytesInPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesInFifteenMinuteRate); assertThat(metrics.getTopicBytesOutPerSec()).containsExactlyEntriesOf(wellKnownMetrics.bytesOutFifteenMinuteRate); // checking per broker io rates assertThat(metrics.getBrokerBytesInPerSec()).containsExactlyInAnyOrderEntriesOf( Map.of(1, new BigDecimal(1), 2, new BigDecimal(10))); assertThat(metrics.getBrokerBytesOutPerSec()).containsExactlyInAnyOrderEntriesOf( Map.of(1, new BigDecimal(2), 2, new BigDecimal(20))); } private void populateWith(Node n, String... prometheusMetric) { Arrays.stream(prometheusMetric) .map(PrometheusEndpointMetricsParser::parse) .filter(Optional::isPresent) .map(Optional::get) .forEach(m -> wellKnownMetrics.populate(n, m)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/AccessControlServiceMock.java ================================================ package com.provectus.kafka.ui.util; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.service.rbac.AccessControlService; import org.mockito.Mockito; import reactor.core.publisher.Mono; public class AccessControlServiceMock { public AccessControlService getMock() { AccessControlService mock = Mockito.mock(AccessControlService.class); when(mock.validateAccess(any())).thenReturn(Mono.empty()); when(mock.isSchemaAccessible(anyString(), anyString())).thenReturn(Mono.just(true)); when(mock.filterViewableTopics(any(), any())).then(invocation -> Mono.just(invocation.getArgument(0))); return mock; } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/DynamicConfigOperationsTest.java ================================================ package com.provectus.kafka.ui.util; import static com.provectus.kafka.ui.util.DynamicConfigOperations.DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY; import static com.provectus.kafka.ui.util.DynamicConfigOperations.DYNAMIC_CONFIG_PATH_ENV_PROPERTY; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.provectus.kafka.ui.config.ClustersProperties; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.MapPropertySource; import org.springframework.core.env.MutablePropertySources; import org.springframework.core.env.PropertySource; class DynamicConfigOperationsTest { private static final String SAMPLE_YAML_CONFIG = """ kafka: clusters: - name: test bootstrapServers: localhost:9092 """; private final ConfigurableApplicationContext ctxMock = mock(ConfigurableApplicationContext.class); private final ConfigurableEnvironment envMock = mock(ConfigurableEnvironment.class); private final DynamicConfigOperations ops = new DynamicConfigOperations(ctxMock); @TempDir private Path tmpDir; @BeforeEach void initMocks() { when(ctxMock.getEnvironment()).thenReturn(envMock); } @Test void initializerAddsDynamicPropertySourceIfAllEnvVarsAreSet() throws Exception { Path propsFilePath = tmpDir.resolve("props.yaml"); Files.writeString(propsFilePath, SAMPLE_YAML_CONFIG, StandardOpenOption.CREATE); MutablePropertySources propertySources = new MutablePropertySources(); propertySources.addFirst(new MapPropertySource("test", Map.of("testK", "testV"))); when(envMock.getPropertySources()).thenReturn(propertySources); mockEnvWithVars(Map.of( DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY, "true", DYNAMIC_CONFIG_PATH_ENV_PROPERTY, propsFilePath.toString() )); DynamicConfigOperations.dynamicConfigPropertiesInitializer().initialize(ctxMock); assertThat(propertySources.size()).isEqualTo(2); assertThat(propertySources.stream()) .element(0) .extracting(PropertySource::getName) .isEqualTo("dynamicProperties"); } @ParameterizedTest @CsvSource({ "false, /tmp/conf.yaml", "true, ", ", /tmp/conf.yaml", ",", "true, /tmp/conf.yaml", //vars set, but file doesn't exist }) void initializerDoNothingIfAnyOfEnvVarsNotSet(@Nullable String enabledVar, @Nullable String pathVar) { var vars = new HashMap(); // using HashMap to keep null values vars.put(DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY, enabledVar); vars.put(DYNAMIC_CONFIG_PATH_ENV_PROPERTY, pathVar); mockEnvWithVars(vars); DynamicConfigOperations.dynamicConfigPropertiesInitializer().initialize(ctxMock); verify(envMock, times(0)).getPropertySources(); } @ParameterizedTest @ValueSource(booleans = {true, false}) void persistRewritesOrCreateConfigFile(boolean exists) throws Exception { Path propsFilePath = tmpDir.resolve("props.yaml"); if (exists) { Files.writeString(propsFilePath, SAMPLE_YAML_CONFIG, StandardOpenOption.CREATE); } mockEnvWithVars(Map.of( DYNAMIC_CONFIG_ENABLED_ENV_PROPERTY, "true", DYNAMIC_CONFIG_PATH_ENV_PROPERTY, propsFilePath.toString() )); var overrideProps = new ClustersProperties(); var cluster = new ClustersProperties.Cluster(); cluster.setName("newName"); overrideProps.setClusters(List.of(cluster)); ops.persist( DynamicConfigOperations.PropertiesStructure.builder() .kafka(overrideProps) .build() ); assertThat(ops.loadDynamicPropertySource()) .get() .extracting(ps -> ps.getProperty("kafka.clusters[0].name")) .isEqualTo("newName"); } private void mockEnvWithVars(Map envVars) { envVars.forEach((k, v) -> when(envMock.getProperty(k)).thenReturn((String) v)); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/GithubReleaseInfoTest.java ================================================ package com.provectus.kafka.ui.util; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; import java.time.Duration; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import reactor.test.StepVerifier; class GithubReleaseInfoTest { private final MockWebServer mockWebServer = new MockWebServer(); @BeforeEach void startMockServer() throws IOException { mockWebServer.start(); } @AfterEach void stopMockServer() throws IOException { mockWebServer.close(); } @Test void test() { mockWebServer.enqueue(new MockResponse() .addHeader("content-type: application/json") .setBody(""" { "published_at": "2023-03-09T16:11:31Z", "tag_name": "v0.6.0", "html_url": "https://github.com/provectus/kafka-ui/releases/tag/v0.6.0", "some_unused_prop": "ololo" } """)); var url = mockWebServer.url("repos/provectus/kafka-ui/releases/latest").toString(); var infoHolder = new GithubReleaseInfo(url); infoHolder.refresh().block(); var i = infoHolder.get(); assertThat(i.html_url()) .isEqualTo("https://github.com/provectus/kafka-ui/releases/tag/v0.6.0"); assertThat(i.published_at()) .isEqualTo("2023-03-09T16:11:31Z"); assertThat(i.tag_name()) .isEqualTo("v0.6.0"); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/PollingThrottlerTest.java ================================================ package com.provectus.kafka.ui.util; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.data.Percentage.withPercentage; import com.google.common.base.Stopwatch; import com.google.common.util.concurrent.RateLimiter; import com.provectus.kafka.ui.emitter.PollingThrottler; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; class PollingThrottlerTest { @Test void testTrafficThrottled() { var throttler = new PollingThrottler("test", RateLimiter.create(1000)); long polledBytes = 0; var stopwatch = Stopwatch.createStarted(); while (stopwatch.elapsed(TimeUnit.SECONDS) < 1) { int newPolled = ThreadLocalRandom.current().nextInt(10); throttler.throttleAfterPoll(newPolled); polledBytes += newPolled; } assertThat(polledBytes).isCloseTo(1000, withPercentage(3.0)); } @Test void noopThrottlerDoNotLimitPolling() { var noopThrottler = PollingThrottler.noop(); var stopwatch = Stopwatch.createStarted(); // emulating that we polled 1GB for (int i = 0; i < 1024; i++) { noopThrottler.throttleAfterPoll(1024 * 1024); } // checking that were are able to "poll" 1GB in less than a second assertThat(stopwatch.elapsed().getSeconds()).isLessThan(1); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/ReactiveFailoverTest.java ================================================ package com.provectus.kafka.ui.util; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.base.Preconditions; import java.time.Duration; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; class ReactiveFailoverTest { private static final String NO_AVAILABLE_PUBLISHERS_MSG = "no active publishers!"; private static final Predicate FAILING_EXCEPTION_FILTER = th -> th.getMessage().contains("fail!"); private static final Supplier FAILING_EXCEPTION_SUPPLIER = () -> new IllegalStateException("fail!"); private static final Duration RETRY_PERIOD = Duration.ofMillis(300); private final List publishers = Stream.generate(Publisher::new).limit(3).toList(); private final ReactiveFailover failover = ReactiveFailover.create( publishers, FAILING_EXCEPTION_FILTER, NO_AVAILABLE_PUBLISHERS_MSG, RETRY_PERIOD ); @Test void testMonoFailoverCycle() throws InterruptedException { // starting with first publisher: // 0 -> ok : ok monoCheck( Map.of( 0, okMono() ), List.of(0), step -> step.expectNextCount(1).verifyComplete() ); // 0 -> fail, 1 -> ok : ok monoCheck( Map.of( 0, failingMono(), 1, okMono() ), List.of(0, 1), step -> step.expectNextCount(1).verifyComplete() ); // 0.failed, 1.failed, 2 -> ok : ok monoCheck( Map.of( 1, failingMono(), 2, okMono() ), List.of(1, 2), step -> step.expectNextCount(1).verifyComplete() ); // 0.failed, 1.failed, 2 -> fail : failing exception monoCheck( Map.of( 2, failingMono() ), List.of(2), step -> step.verifyErrorMessage(FAILING_EXCEPTION_SUPPLIER.get().getMessage()) ); // 0.failed, 1.failed, 2.failed : No alive publisher exception monoCheck( Map.of(), List.of(), step -> step.verifyErrorMessage(NO_AVAILABLE_PUBLISHERS_MSG) ); // resetting retry: all publishers became alive: 0.ok, 1.ok, 2.ok Thread.sleep(RETRY_PERIOD.toMillis() + 1); // starting with last errored publisher: // 2 -> fail, 0 -> fail, 1 -> ok : ok monoCheck( Map.of( 2, failingMono(), 0, failingMono(), 1, okMono() ), List.of(2, 0, 1), step -> step.expectNextCount(1).verifyComplete() ); // 1 -> ok : ok monoCheck( Map.of( 1, okMono() ), List.of(1), step -> step.expectNextCount(1).verifyComplete() ); } @Test void testFluxFailoverCycle() throws InterruptedException { // starting with first publisher: // 0 -> ok : ok fluxCheck( Map.of( 0, okFlux() ), List.of(0), step -> step.expectNextCount(1).verifyComplete() ); // 0 -> fail, 1 -> ok : ok fluxCheck( Map.of( 0, failingFlux(), 1, okFlux() ), List.of(0, 1), step -> step.expectNextCount(1).verifyComplete() ); // 0.failed, 1.failed, 2 -> ok : ok fluxCheck( Map.of( 1, failingFlux(), 2, okFlux() ), List.of(1, 2), step -> step.expectNextCount(1).verifyComplete() ); // 0.failed, 1.failed, 2 -> fail : failing exception fluxCheck( Map.of( 2, failingFlux() ), List.of(2), step -> step.verifyErrorMessage(FAILING_EXCEPTION_SUPPLIER.get().getMessage()) ); // 0.failed, 1.failed, 2.failed : No alive publisher exception fluxCheck( Map.of(), List.of(), step -> step.verifyErrorMessage(NO_AVAILABLE_PUBLISHERS_MSG) ); // resetting retry: all publishers became alive: 0.ok, 1.ok, 2.ok Thread.sleep(RETRY_PERIOD.toMillis() + 1); // starting with last errored publisher: // 2 -> fail, 0 -> fail, 1 -> ok : ok fluxCheck( Map.of( 2, failingFlux(), 0, failingFlux(), 1, okFlux() ), List.of(2, 0, 1), step -> step.expectNextCount(1).verifyComplete() ); // 1 -> ok : ok fluxCheck( Map.of( 1, okFlux() ), List.of(1), step -> step.expectNextCount(1).verifyComplete() ); } private void monoCheck(Map> mock, List publishersToBeCalled, // for checking calls order Consumer> stepVerifier) { AtomicInteger calledCount = new AtomicInteger(); var mono = failover.mono(publisher -> { int calledPublisherIdx = publishers.indexOf(publisher); assertThat(calledPublisherIdx).isEqualTo(publishersToBeCalled.get(calledCount.getAndIncrement())); return Preconditions.checkNotNull( mock.get(calledPublisherIdx), "Mono result not set for publisher %d", calledPublisherIdx ); }); stepVerifier.accept(StepVerifier.create(mono)); assertThat(calledCount.get()).isEqualTo(publishersToBeCalled.size()); } private void fluxCheck(Map> mock, List publishersToBeCalled, // for checking calls order Consumer> stepVerifier) { AtomicInteger calledCount = new AtomicInteger(); var flux = failover.flux(publisher -> { int calledPublisherIdx = publishers.indexOf(publisher); assertThat(calledPublisherIdx).isEqualTo(publishersToBeCalled.get(calledCount.getAndIncrement())); return Preconditions.checkNotNull( mock.get(calledPublisherIdx), "Mono result not set for publisher %d", calledPublisherIdx ); }); stepVerifier.accept(StepVerifier.create(flux)); assertThat(calledCount.get()).isEqualTo(publishersToBeCalled.size()); } private Flux okFlux() { return Flux.just("ok"); } private Flux failingFlux() { return Flux.error(FAILING_EXCEPTION_SUPPLIER); } private Mono okMono() { return Mono.just("ok"); } private Mono failingMono() { return Mono.error(FAILING_EXCEPTION_SUPPLIER); } public static class Publisher { } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/jsonschema/AvroJsonSchemaConverterTest.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.ObjectMapper; import java.net.URI; import java.net.URISyntaxException; import lombok.SneakyThrows; import org.apache.avro.Schema; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class AvroJsonSchemaConverterTest { private AvroJsonSchemaConverter converter; private URI basePath; @BeforeEach void init() throws URISyntaxException { converter = new AvroJsonSchemaConverter(); basePath = new URI("http://example.com/"); } @Test void avroConvertTest() { String avroSchema = " {" + " \"type\": \"record\"," + " \"name\": \"Message\"," + " \"namespace\": \"com.provectus.kafka\"," + " \"fields\": [" + " {" + " \"name\": \"record\"," + " \"type\": {" + " \"type\": \"record\"," + " \"name\": \"InnerMessage\"," + " \"fields\": [" + " {" + " \"name\": \"id\"," + " \"type\": \"long\"" + " }," + " {" + " \"name\": \"text\"," + " \"type\": \"string\"" + " }," + " {" + " \"name\": \"long_text\"," + " \"type\": [" + " \"null\"," + " \"string\"" + " ]," + " \"default\": null" + " }," + " {" + " \"name\": \"order\"," + " \"type\": {" + " \"type\": \"enum\"," + " \"name\": \"Suit\"," + " \"symbols\": [\"SPADES\",\"HEARTS\",\"DIAMONDS\",\"CLUBS\"]" + " }" + " }," + " {" + " \"name\": \"array\"," + " \"type\": {" + " \"type\": \"array\"," + " \"items\": \"string\"," + " \"default\": []" + " }" + " }," + " {" + " \"name\": \"map\"," + " \"type\": {" + " \"type\": \"map\"," + " \"values\": \"long\"," + " \"default\": {}" + " }" + " }" + " ]" + " }" + " }" + " ]" + " }"; String expectedJsonSchema = "{ " + " \"$id\" : \"http://example.com/Message\", " + " \"$schema\" : \"https://json-schema.org/draft/2020-12/schema\", " + " \"type\" : \"object\", " + " \"properties\" : { " + " \"record\" : { \"$ref\" : \"#/definitions/com.provectus.kafka.InnerMessage\" } " + " }, " + " \"required\" : [ \"record\" ], " + " \"definitions\" : { " + " \"com.provectus.kafka.Message\" : { \"$ref\" : \"#\" }, " + " \"com.provectus.kafka.InnerMessage\" : { " + " \"type\" : \"object\", " + " \"properties\" : { " + " \"long_text\" : { " + " \"oneOf\" : [ { " + " \"type\" : \"null\" " + " }, { " + " \"type\" : \"object\", " + " \"properties\" : { " + " \"string\" : { " + " \"type\" : \"string\" " + " } " + " } " + " } ] " + " }, " + " \"array\" : { " + " \"type\" : \"array\", " + " \"items\" : { \"type\" : \"string\" } " + " }, " + " \"id\" : { \"type\" : \"integer\" }, " + " \"text\" : { \"type\" : \"string\" }, " + " \"map\" : { " + " \"type\" : \"object\", " + " \"additionalProperties\" : { \"type\" : \"integer\" } " + " }, " + " \"order\" : { " + " \"enum\" : [ \"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\" ], " + " \"type\" : \"string\" " + " } " + " }, " + " \"required\" : [ \"id\", \"text\", \"order\", \"array\", \"map\" ] " + " } " + " } " + "}"; convertAndCompare(expectedJsonSchema, avroSchema); } @Test void testNullableUnions() { String avroSchema = " {" + " \"type\": \"record\"," + " \"name\": \"Message\"," + " \"namespace\": \"com.provectus.kafka\"," + " \"fields\": [" + " {" + " \"name\": \"text\"," + " \"type\": [" + " \"null\"," + " \"string\"" + " ]," + " \"default\": null" + " }," + " {" + " \"name\": \"value\"," + " \"type\": [" + " \"null\"," + " \"string\"," + " \"long\"" + " ]," + " \"default\": null" + " }" + " ]" + " }"; String expectedJsonSchema = "{\"$id\":\"http://example.com/Message\"," + "\"$schema\":\"https://json-schema.org/draft/2020-12/schema\"," + "\"type\":\"object\",\"properties\":{\"text\":" + "{\"oneOf\":[{\"type\":\"null\"},{\"type\":\"object\"," + "\"properties\":{\"string\":{\"type\":\"string\"}}}]},\"value\":" + "{\"oneOf\":[{\"type\":\"null\"},{\"type\":\"object\"," + "\"properties\":{\"string\":{\"type\":\"string\"},\"long\":{\"type\":\"integer\"}}}]}}," + "\"definitions\" : { \"com.provectus.kafka.Message\" : { \"$ref\" : \"#\" }}}"; convertAndCompare(expectedJsonSchema, avroSchema); } @Test void testRecordReferences() { String avroSchema = "{\n" + " \"type\": \"record\", " + " \"namespace\": \"n.s\", " + " \"name\": \"RootMsg\", " + " \"fields\":\n" + " [ " + " { " + " \"name\": \"inner1\", " + " \"type\": { " + " \"type\": \"record\", " + " \"name\": \"Inner\", " + " \"fields\": [ { \"name\": \"f1\", \"type\": \"double\" } ] " + " } " + " }, " + " { " + " \"name\": \"inner2\", " + " \"type\": { " + " \"type\": \"record\", " + " \"namespace\": \"n.s2\", " + " \"name\": \"Inner\", " + " \"fields\": " + " [ { \"name\": \"f1\", \"type\": \"double\" } ] " + " } " + " }, " + " { " + " \"name\": \"refField\", " + " \"type\": [ \"null\", \"Inner\", \"n.s2.Inner\", \"RootMsg\" ] " + " } " + " ] " + "}"; String expectedJsonSchema = "{ " + " \"$id\" : \"http://example.com/RootMsg\", " + " \"$schema\" : \"https://json-schema.org/draft/2020-12/schema\", " + " \"type\" : \"object\", " + " \"properties\" : { " + " \"inner1\" : { \"$ref\" : \"#/definitions/n.s.Inner\" }, " + " \"inner2\" : { \"$ref\" : \"#/definitions/n.s2.Inner\" }, " + " \"refField\" : { " + " \"oneOf\" : [ " + " { " + " \"type\" : \"null\" " + " }, " + " { " + " \"type\" : \"object\", " + " \"properties\" : { " + " \"n.s.RootMsg\" : { \"$ref\" : \"#/definitions/n.s.RootMsg\" }, " + " \"n.s2.Inner\" : { \"$ref\" : \"#/definitions/n.s2.Inner\" }, " + " \"n.s.Inner\" : { \"$ref\" : \"#/definitions/n.s.Inner\" } " + " } " + " } ] " + " } " + " }, " + " \"required\" : [ \"inner1\", \"inner2\" ], " + " \"definitions\" : { " + " \"n.s.RootMsg\" : { \"$ref\" : \"#\" }, " + " \"n.s2.Inner\" : { " + " \"type\" : \"object\", " + " \"properties\" : { \"f1\" : { \"type\" : \"number\" } }, " + " \"required\" : [ \"f1\" ] " + " }, " + " \"n.s.Inner\" : { " + " \"type\" : \"object\", " + " \"properties\" : { \"f1\" : { \"type\" : \"number\" } }, " + " \"required\" : [ \"f1\" ] " + " } " + " } " + "}"; convertAndCompare(expectedJsonSchema, avroSchema); } @SneakyThrows private void convertAndCompare(String expectedJsonSchema, String sourceAvroSchema) { var parseAvroSchema = new Schema.Parser().parse(sourceAvroSchema); var converted = converter.convert(basePath, parseAvroSchema).toJson(); var objectMapper = new ObjectMapper(); Assertions.assertEquals( objectMapper.readTree(expectedJsonSchema), objectMapper.readTree(converted) ); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/jsonschema/JsonAvroConversionTest.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import static com.provectus.kafka.ui.util.jsonschema.JsonAvroConversion.convertAvroToJson; import static com.provectus.kafka.ui.util.jsonschema.JsonAvroConversion.convertJsonToAvro; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DoubleNode; import com.fasterxml.jackson.databind.node.FloatNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.LongNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.primitives.Longs; import com.provectus.kafka.ui.exception.JsonAvroConversionException; import io.confluent.kafka.schemaregistry.avro.AvroSchema; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.util.List; import java.util.Map; import java.util.UUID; import lombok.SneakyThrows; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class JsonAvroConversionTest { // checking conversion from json to KafkaAvroSerializer-compatible avro objects @Nested class FromJsonToAvro { @Test void primitiveRoot() { assertThat(convertJsonToAvro("\"str\"", createSchema("\"string\""))) .isEqualTo("str"); assertThat(convertJsonToAvro("123", createSchema("\"int\""))) .isEqualTo(123); assertThat(convertJsonToAvro("123", createSchema("\"long\""))) .isEqualTo(123L); assertThat(convertJsonToAvro("123.123", createSchema("\"float\""))) .isEqualTo(123.123F); assertThat(convertJsonToAvro("12345.12345", createSchema("\"double\""))) .isEqualTo(12345.12345); } @Test void primitiveTypedFields() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "f_int", "type": "int" }, { "name": "f_long", "type": "long" }, { "name": "f_string", "type": "string" }, { "name": "f_boolean", "type": "boolean" }, { "name": "f_float", "type": "float" }, { "name": "f_double", "type": "double" }, { "name": "f_enum", "type" : { "type": "enum", "name": "Suit", "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] } }, { "name" : "f_fixed", "type" : { "type" : "fixed" ,"size" : 8, "name": "long_encoded" } }, { "name" : "f_bytes", "type": "bytes" } ] }""" ); String jsonPayload = """ { "f_int": 123, "f_long": 4294967294, "f_string": "string here", "f_boolean": true, "f_float": 123.1, "f_double": 123456.123456, "f_enum": "SPADES", "f_fixed": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0004Ò", "f_bytes": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\t)" } """; var converted = convertJsonToAvro(jsonPayload, schema); assertThat(converted).isInstanceOf(GenericData.Record.class); var record = (GenericData.Record) converted; assertThat(record.get("f_int")).isEqualTo(123); assertThat(record.get("f_long")).isEqualTo(4294967294L); assertThat(record.get("f_string")).isEqualTo("string here"); assertThat(record.get("f_boolean")).isEqualTo(true); assertThat(record.get("f_float")).isEqualTo(123.1f); assertThat(record.get("f_double")).isEqualTo(123456.123456); assertThat(record.get("f_enum")) .isEqualTo( new GenericData.EnumSymbol( schema.getField("f_enum").schema(), "SPADES" ) ); assertThat(((GenericData.Fixed) record.get("f_fixed")).bytes()).isEqualTo(Longs.toByteArray(1234L)); assertThat(((ByteBuffer) record.get("f_bytes")).array()).isEqualTo(Longs.toByteArray(2345L)); } @Test void unionRoot() { var schema = createSchema("[ \"null\", \"string\", \"int\" ]"); var converted = convertJsonToAvro("{\"string\":\"string here\"}", schema); assertThat(converted).isEqualTo("string here"); converted = convertJsonToAvro("{\"int\": 123}", schema); assertThat(converted).isEqualTo(123); converted = convertJsonToAvro("null", schema); assertThat(converted).isEqualTo(null); } @Test void unionField() { var schema = createSchema( """ { "type": "record", "namespace": "com.test", "name": "TestAvroRecord", "fields": [ { "name": "f_union", "type": [ "null", "int", "TestAvroRecord"] } ] }""" ); String jsonPayload = "{ \"f_union\": null }"; var record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_union")).isNull(); jsonPayload = "{ \"f_union\": { \"int\": 123 } }"; record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_union")).isEqualTo(123); //short name can be used since there is no clash with other type names jsonPayload = "{ \"f_union\": { \"TestAvroRecord\": { \"f_union\": { \"int\": 123 } } } }"; record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_union")).isInstanceOf(GenericData.Record.class); var innerRec = (GenericData.Record) record.get("f_union"); assertThat(innerRec.get("f_union")).isEqualTo(123); assertThatThrownBy(() -> convertJsonToAvro("{ \"f_union\": { \"NotExistingType\": 123 } }", schema) ).isInstanceOf(JsonAvroConversionException.class); } @Test void unionFieldWithTypeNamesClash() { var schema = createSchema( """ { "type": "record", "namespace": "com.test", "name": "TestAvroRecord", "fields": [ { "name": "nestedClass", "type": { "type": "record", "namespace": "com.nested", "name": "TestAvroRecord", "fields": [ {"name" : "inner_obj_field", "type": "int" } ] } }, { "name": "f_union", "type": [ "null", "int", "com.test.TestAvroRecord", "com.nested.TestAvroRecord"] } ] }""" ); //short name can't can be used since there is a clash with other type names var jsonPayload = "{ \"f_union\": { \"com.test.TestAvroRecord\": { \"f_union\": { \"int\": 123 } } } }"; var record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_union")).isInstanceOf(GenericData.Record.class); var innerRec = (GenericData.Record) record.get("f_union"); assertThat(innerRec.get("f_union")).isEqualTo(123); //short name can't can be used since there is a clash with other type names jsonPayload = "{ \"f_union\": { \"com.nested.TestAvroRecord\": { \"inner_obj_field\": 234 } } }"; record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_union")).isInstanceOf(GenericData.Record.class); innerRec = (GenericData.Record) record.get("f_union"); assertThat(innerRec.get("inner_obj_field")).isEqualTo(234); assertThatThrownBy(() -> convertJsonToAvro("{ \"f_union\": { \"TestAvroRecord\": { \"inner_obj_field\": 234 } } }", schema) ).isInstanceOf(JsonAvroConversionException.class); } @Test void mapField() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "long_map", "type": { "type": "map", "values" : "long", "default": {} } }, { "name": "string_map", "type": { "type": "map", "values" : "string", "default": {} } }, { "name": "self_ref_map", "type": { "type": "map", "values" : "TestAvroRecord", "default": {} } } ] }""" ); String jsonPayload = """ { "long_map": { "k1": 123, "k2": 456 }, "string_map": { "k3": "s1", "k4": "s2" }, "self_ref_map": { "k5" : { "long_map": { "_k1": 222 }, "string_map": { "_k2": "_s1" } } } } """; var record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("long_map")) .isEqualTo(Map.of("k1", 123L, "k2", 456L)); assertThat(record.get("string_map")) .isEqualTo(Map.of("k3", "s1", "k4", "s2")); assertThat(record.get("self_ref_map")) .isNotNull(); Map selfRefMapField = (Map) record.get("self_ref_map"); assertThat(selfRefMapField) .hasSize(1) .hasEntrySatisfying("k5", v -> { assertThat(v).isInstanceOf(GenericData.Record.class); var innerRec = (GenericData.Record) v; assertThat(innerRec.get("long_map")) .isEqualTo(Map.of("_k1", 222L)); assertThat(innerRec.get("string_map")) .isEqualTo(Map.of("_k2", "_s1")); }); } @Test void arrayField() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "f_array", "type": { "type": "array", "items" : "string", "default": [] } } ] }""" ); String jsonPayload = """ { "f_array": [ "e1", "e2" ] } """; var record = (GenericData.Record) convertJsonToAvro(jsonPayload, schema); assertThat(record.get("f_array")).isEqualTo(List.of("e1", "e2")); } @Test void logicalTypesField() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "lt_date", "type": { "type": "int", "logicalType": "date" } }, { "name": "lt_uuid", "type": { "type": "string", "logicalType": "uuid" } }, { "name": "lt_decimal", "type": { "type": "bytes", "logicalType": "decimal", "precision": 22, "scale":10 } }, { "name": "lt_time_millis", "type": { "type": "int", "logicalType": "time-millis"} }, { "name": "lt_time_micros", "type": { "type": "long", "logicalType": "time-micros"} }, { "name": "lt_timestamp_millis", "type": { "type": "long", "logicalType": "timestamp-millis" } }, { "name": "lt_timestamp_micros", "type": { "type": "long", "logicalType": "timestamp-micros" } }, { "name": "lt_local_timestamp_millis", "type": { "type": "long", "logicalType": "local-timestamp-millis" } }, { "name": "lt_local_timestamp_micros", "type": { "type": "long", "logicalType": "local-timestamp-micros" } } ] }""" ); String jsonPayload = """ { "lt_date":"1991-08-14", "lt_decimal": 2.1617413862327545E11, "lt_time_millis": "10:15:30.001", "lt_time_micros": "10:15:30.123456", "lt_uuid": "a37b75ca-097c-5d46-6119-f0637922e908", "lt_timestamp_millis": "2007-12-03T10:15:30.123Z", "lt_timestamp_micros": "2007-12-13T10:15:30.123456Z", "lt_local_timestamp_millis": "2017-12-03T10:15:30.123", "lt_local_timestamp_micros": "2017-12-13T10:15:30.123456" } """; var converted = convertJsonToAvro(jsonPayload, schema); assertThat(converted).isInstanceOf(GenericData.Record.class); var record = (GenericData.Record) converted; assertThat(record.get("lt_date")) .isEqualTo(LocalDate.of(1991, 8, 14)); assertThat(record.get("lt_decimal")) .isEqualTo(new BigDecimal("2.1617413862327545E11")); assertThat(record.get("lt_time_millis")) .isEqualTo(LocalTime.parse("10:15:30.001")); assertThat(record.get("lt_time_micros")) .isEqualTo(LocalTime.parse("10:15:30.123456")); assertThat(record.get("lt_timestamp_millis")) .isEqualTo(Instant.parse("2007-12-03T10:15:30.123Z")); assertThat(record.get("lt_timestamp_micros")) .isEqualTo(Instant.parse("2007-12-13T10:15:30.123456Z")); assertThat(record.get("lt_local_timestamp_millis")) .isEqualTo(LocalDateTime.parse("2017-12-03T10:15:30.123")); assertThat(record.get("lt_local_timestamp_micros")) .isEqualTo(LocalDateTime.parse("2017-12-13T10:15:30.123456")); } } // checking conversion of KafkaAvroDeserializer output to JsonNode @Nested class FromAvroToJson { @Test void primitiveRoot() { assertThat(convertAvroToJson("str", createSchema("\"string\""))) .isEqualTo(new TextNode("str")); assertThat(convertAvroToJson(123, createSchema("\"int\""))) .isEqualTo(new IntNode(123)); assertThat(convertAvroToJson(123L, createSchema("\"long\""))) .isEqualTo(new LongNode(123)); assertThat(convertAvroToJson(123.1F, createSchema("\"float\""))) .isEqualTo(new FloatNode(123.1F)); assertThat(convertAvroToJson(123.1, createSchema("\"double\""))) .isEqualTo(new DoubleNode(123.1)); assertThat(convertAvroToJson(true, createSchema("\"boolean\""))) .isEqualTo(BooleanNode.valueOf(true)); assertThat(convertAvroToJson(ByteBuffer.wrap(Longs.toByteArray(123L)), createSchema("\"bytes\""))) .isEqualTo(new TextNode(new String(Longs.toByteArray(123L), StandardCharsets.ISO_8859_1))); } @SneakyThrows @Test void primitiveTypedFields() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "f_int", "type": "int" }, { "name": "f_long", "type": "long" }, { "name": "f_string", "type": "string" }, { "name": "f_boolean", "type": "boolean" }, { "name": "f_float", "type": "float" }, { "name": "f_double", "type": "double" }, { "name": "f_enum", "type" : { "type": "enum", "name": "Suit", "symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"] } }, { "name" : "f_fixed", "type" : { "type" : "fixed" ,"size" : 8, "name": "long_encoded" } }, { "name" : "f_bytes", "type": "bytes" } ] }""" ); byte[] fixedFieldValue = Longs.toByteArray(1234L); byte[] bytesFieldValue = Longs.toByteArray(2345L); GenericData.Record inputRecord = new GenericData.Record(schema); inputRecord.put("f_int", 123); inputRecord.put("f_long", 4294967294L); inputRecord.put("f_string", "string here"); inputRecord.put("f_boolean", true); inputRecord.put("f_float", 123.1f); inputRecord.put("f_double", 123456.123456); inputRecord.put("f_enum", new GenericData.EnumSymbol(schema.getField("f_enum").schema(), "SPADES")); inputRecord.put("f_fixed", new GenericData.Fixed(schema.getField("f_fixed").schema(), fixedFieldValue)); inputRecord.put("f_bytes", ByteBuffer.wrap(bytesFieldValue)); String expectedJson = """ { "f_int": 123, "f_long": 4294967294, "f_string": "string here", "f_boolean": true, "f_float": 123.1, "f_double": 123456.123456, "f_enum": "SPADES", "f_fixed": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0004Ò", "f_bytes": "\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\t)" } """; assertJsonsEqual(expectedJson, convertAvroToJson(inputRecord, schema)); } @Test void logicalTypesField() { var schema = createSchema( """ { "type": "record", "name": "TestAvroRecord", "fields": [ { "name": "lt_date", "type": { "type": "int", "logicalType": "date" } }, { "name": "lt_uuid", "type": { "type": "string", "logicalType": "uuid" } }, { "name": "lt_decimal", "type": { "type": "bytes", "logicalType": "decimal", "precision": 22, "scale":10 } }, { "name": "lt_time_millis", "type": { "type": "int", "logicalType": "time-millis"} }, { "name": "lt_time_micros", "type": { "type": "long", "logicalType": "time-micros"} }, { "name": "lt_timestamp_millis", "type": { "type": "long", "logicalType": "timestamp-millis" } }, { "name": "lt_timestamp_micros", "type": { "type": "long", "logicalType": "timestamp-micros" } }, { "name": "lt_local_timestamp_millis", "type": { "type": "long", "logicalType": "local-timestamp-millis" } }, { "name": "lt_local_timestamp_micros", "type": { "type": "long", "logicalType": "local-timestamp-micros" } } ] }""" ); GenericData.Record inputRecord = new GenericData.Record(schema); inputRecord.put("lt_date", LocalDate.of(1991, 8, 14)); inputRecord.put("lt_uuid", UUID.fromString("a37b75ca-097c-5d46-6119-f0637922e908")); inputRecord.put("lt_decimal", new BigDecimal("2.16")); inputRecord.put("lt_time_millis", LocalTime.parse("10:15:30.001")); inputRecord.put("lt_time_micros", LocalTime.parse("10:15:30.123456")); inputRecord.put("lt_timestamp_millis", Instant.parse("2007-12-03T10:15:30.123Z")); inputRecord.put("lt_timestamp_micros", Instant.parse("2007-12-13T10:15:30.123456Z")); inputRecord.put("lt_local_timestamp_millis", LocalDateTime.parse("2017-12-03T10:15:30.123")); inputRecord.put("lt_local_timestamp_micros", LocalDateTime.parse("2017-12-13T10:15:30.123456")); String expectedJson = """ { "lt_date":"1991-08-14", "lt_uuid": "a37b75ca-097c-5d46-6119-f0637922e908", "lt_decimal": 2.16, "lt_time_millis": "10:15:30.001", "lt_time_micros": "10:15:30.123456", "lt_timestamp_millis": "2007-12-03T10:15:30.123Z", "lt_timestamp_micros": "2007-12-13T10:15:30.123456Z", "lt_local_timestamp_millis": "2017-12-03T10:15:30.123", "lt_local_timestamp_micros": "2017-12-13T10:15:30.123456" } """; assertJsonsEqual(expectedJson, convertAvroToJson(inputRecord, schema)); } @Test void unionField() { var schema = createSchema( """ { "type": "record", "namespace": "com.test", "name": "TestAvroRecord", "fields": [ { "name": "f_union", "type": [ "null", "int", "TestAvroRecord"] } ] }""" ); var r = new GenericData.Record(schema); r.put("f_union", null); assertJsonsEqual(" {}", convertAvroToJson(r, schema)); r = new GenericData.Record(schema); r.put("f_union", 123); assertJsonsEqual(" { \"f_union\" : { \"int\" : 123 } }", convertAvroToJson(r, schema)); r = new GenericData.Record(schema); var innerRec = new GenericData.Record(schema); innerRec.put("f_union", 123); r.put("f_union", innerRec); // short type name can be set since there is NO clash with other types name assertJsonsEqual( " { \"f_union\" : { \"TestAvroRecord\" : { \"f_union\" : { \"int\" : 123 } } } }", convertAvroToJson(r, schema) ); } @Test void unionFieldWithInnerTypesNamesClash() { var schema = createSchema( """ { "type": "record", "namespace": "com.test", "name": "TestAvroRecord", "fields": [ { "name": "nestedClass", "type": { "type": "record", "namespace": "com.nested", "name": "TestAvroRecord", "fields": [ {"name" : "inner_obj_field", "type": "int" } ] } }, { "name": "f_union", "type": [ "null", "int", "com.test.TestAvroRecord", "com.nested.TestAvroRecord"] } ] }""" ); var r = new GenericData.Record(schema); var innerRec = new GenericData.Record(schema); innerRec.put("f_union", 123); r.put("f_union", innerRec); // full type name should be set since there is a clash with other type name assertJsonsEqual( " { \"f_union\" : { \"com.test.TestAvroRecord\" : { \"f_union\" : { \"int\" : 123 } } } }", convertAvroToJson(r, schema) ); } } private Schema createSchema(String schema) { return new AvroSchema(schema).rawSchema(); } @SneakyThrows private void assertJsonsEqual(String expectedJson, JsonNode actual) { var mapper = new JsonMapper(); assertThat(actual.toPrettyString()) .isEqualTo(mapper.readTree(expectedJson).toPrettyString()); } } ================================================ FILE: kafka-ui-api/src/test/java/com/provectus/kafka/ui/util/jsonschema/ProtobufSchemaConverterTest.java ================================================ package com.provectus.kafka.ui.util.jsonschema; import com.fasterxml.jackson.databind.ObjectMapper; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import java.net.URI; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; class ProtobufSchemaConverterTest { @Test void testSchemaConvert() throws Exception { String protoSchema = """ syntax = "proto3"; package test; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; message TestMsg { string string_field = 1; int32 int32_field = 2; bool bool_field = 3; SampleEnum enum_field = 4; enum SampleEnum { ENUM_V1 = 0; ENUM_V2 = 1; } google.protobuf.Timestamp ts_field = 5; google.protobuf.Struct struct_field = 6; google.protobuf.ListValue lst_v_field = 7; google.protobuf.Duration duration_field = 8; oneof some_oneof1 { google.protobuf.Value v1 = 9; google.protobuf.Value v2 = 10; } // wrapper fields: google.protobuf.Int64Value int64_w_field = 11; google.protobuf.Int32Value int32_w_field = 12; google.protobuf.UInt64Value uint64_w_field = 13; google.protobuf.UInt32Value uint32_w_field = 14; google.protobuf.StringValue string_w_field = 15; google.protobuf.BoolValue bool_w_field = 16; google.protobuf.DoubleValue double_w_field = 17; google.protobuf.FloatValue float_w_field = 18; //embedded msg EmbeddedMsg emb = 19; repeated EmbeddedMsg emb_list = 20; message EmbeddedMsg { int32 emb_f1 = 1; TestMsg outer_ref = 2; EmbeddedMsg self_ref = 3; } map intToStringMap = 21; map strToObjMap = 22; }"""; String expectedJsonSchema = """ { "$id": "http://example.com/test.TestMsg", "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "definitions": { "test.TestMsg": { "type": "object", "properties": { "enum_field": { "enum": [ "ENUM_V1", "ENUM_V2" ], "type": "string" }, "string_w_field": { "type": "string" }, "ts_field": { "type": "string", "format": "date-time" }, "emb_list": { "type": "array", "items": { "$ref": "#/definitions/test.TestMsg.EmbeddedMsg" } }, "float_w_field": { "type": "number" }, "lst_v_field": { "type": "array", "items": { "type":[ "number", "string", "object", "array", "boolean", "null" ] } }, "struct_field": { "type": "object", "properties": {} }, "string_field": { "type": "string" }, "double_w_field": { "type": "number" }, "bool_field": { "type": "boolean" }, "int32_w_field": { "type": "integer", "maximum": 2147483647, "minimum": -2147483648 }, "duration_field": { "type": "string" }, "int32_field": { "type": "integer", "maximum": 2147483647, "minimum": -2147483648 }, "int64_w_field": { "type": "integer", "maximum": 9223372036854775807, "minimum": -9223372036854775808 }, "v1": { "type": [ "number", "string", "object", "array", "boolean", "null" ] }, "emb": { "$ref": "#/definitions/test.TestMsg.EmbeddedMsg" }, "v2": { "type": [ "number", "string", "object", "array", "boolean", "null" ] }, "uint32_w_field": { "type": "integer", "maximum": 4294967295, "minimum": 0 }, "bool_w_field": { "type": "boolean" }, "uint64_w_field": { "type": "integer", "maximum": 18446744073709551615, "minimum": 0 }, "strToObjMap": { "type": "object", "additionalProperties": true }, "intToStringMap": { "type": "object", "additionalProperties": true } } }, "test.TestMsg.EmbeddedMsg": { "type": "object", "properties": { "emb_f1": { "type": "integer", "maximum": 2147483647, "minimum": -2147483648 }, "outer_ref": { "$ref": "#/definitions/test.TestMsg" }, "self_ref": { "$ref": "#/definitions/test.TestMsg.EmbeddedMsg" } } } }, "$ref": "#/definitions/test.TestMsg" }"""; ProtobufSchemaConverter converter = new ProtobufSchemaConverter(); ProtobufSchema protobufSchema = new ProtobufSchema(protoSchema); URI basePath = new URI("http://example.com/"); JsonSchema converted = converter.convert(basePath, protobufSchema.toDescriptor()); assertJsonEqual(expectedJsonSchema, converted.toJson()); } private void assertJsonEqual(String expected, String actual) throws Exception { ObjectMapper om = new ObjectMapper(); Assertions.assertEquals(om.readTree(expected), om.readTree(actual)); } } ================================================ FILE: kafka-ui-api/src/test/resources/application-test.yml ================================================ spring: jmx: enabled: true auth: type: DISABLED ================================================ FILE: kafka-ui-api/src/test/resources/fileForUploadTest.txt ================================================ some content goes here ================================================ FILE: kafka-ui-api/src/test/resources/protobuf-serde/address-book.proto ================================================ syntax = "proto3"; package test; option java_multiple_files = true; option java_package = "com.example.tutorial.protos"; option java_outer_classname = "AddressBookProtos"; message Person { string name = 1; int32 id = 2; // Unique ID number for this person. string email = 3; enum PhoneType { MOBILE = 0; HOME = 1; WORK = 2; } message PhoneNumber { string number = 1; PhoneType type = 2; } repeated PhoneNumber phones = 4; } message AnotherPerson { string name = 1; string surname = 2; } // Our address book file is just one of these. message AddressBook { int32 version = 1; repeated Person people = 2; } ================================================ FILE: kafka-ui-api/src/test/resources/protobuf-serde/lang-description.proto ================================================ syntax = "proto3"; package test; import "language/language.proto"; import "google/protobuf/wrappers.proto"; message LanguageDescription { test.lang.Language lang = 1; google.protobuf.StringValue descr = 2; } ================================================ FILE: kafka-ui-api/src/test/resources/protobuf-serde/language/language.proto ================================================ syntax = "proto3"; package test.lang; enum Language { DE = 0; EN = 1; ES = 2; FR = 3; PL = 4; RU = 5; } ================================================ FILE: kafka-ui-api/src/test/resources/protobuf-serde/sensor.proto ================================================ syntax = "proto3"; package test; message Sensor { string name = 1; double temperature = 2; int32 humidity = 3; enum SwitchLevel { CLOSED = 0; OPEN = 1; } SwitchLevel door = 5; } ================================================ FILE: kafka-ui-contract/pom.xml ================================================ kafka-ui com.provectus 0.0.1-SNAPSHOT 4.0.0 kafka-ui-contract generate-spring-webflux-api true org.springframework.boot spring-boot-starter-webflux org.springframework.boot spring-boot-starter-validation io.swagger.core.v3 swagger-integration-jakarta 2.2.8 org.openapitools jackson-databind-nullable 0.2.4 jakarta.annotation jakarta.annotation-api 2.1.1 javax.annotation javax.annotation-api 1.3.2 org.openapitools openapi-generator-maven-plugin ${openapi-generator-maven-plugin.version} generate-kafka-ui-client generate ${project.basedir}/src/main/resources/swagger/kafka-ui-api.yaml ${project.build.directory}/generated-sources/kafka-ui-client java false false com.provectus.kafka.ui.api.model com.provectus.kafka.ui.api.api kafka-ui-client true webclient true java8 true generate-backend-api generate ${project.basedir}/src/main/resources/swagger/kafka-ui-api.yaml ${project.build.directory}/generated-sources/api spring DTO com.provectus.kafka.ui.model com.provectus.kafka.ui.api kafka-ui-contract true true true true true true java8 generate-connect-client generate ${project.basedir}/src/main/resources/swagger/kafka-connect-api.yaml ${project.build.directory}/generated-sources/kafka-connect-client java false false com.provectus.kafka.ui.connect.model com.provectus.kafka.ui.connect.api kafka-connect-client true webclient true true java8 generate-sr-client generate ${project.basedir}/src/main/resources/swagger/kafka-sr-api.yaml ${project.build.directory}/generated-sources/kafka-sr-client java false false com.provectus.kafka.ui.sr.model com.provectus.kafka.ui.sr.api kafka-sr-client true webclient true true java8 com.github.eirslett frontend-maven-plugin ${frontend-maven-plugin.version} ../kafka-ui-react-app ${project.version} install node and pnpm install-node-and-pnpm ${node.version} ${pnpm.version} pnpm install pnpm install pnpm gen:sources pnpm gen:sources org.apache.maven.plugins maven-clean-plugin ${basedir}/${frontend-generated-sources-directory} org.apache.maven.plugins maven-resources-plugin copy-resource-one generate-resources copy-resources ${basedir}/${frontend-generated-sources-directory} ${project.build.directory}/generated-sources/frontend/ **/*.ts ================================================ FILE: kafka-ui-contract/src/main/resources/swagger/kafka-connect-api.yaml ================================================ openapi: 3.0.0 info: description: Api Documentation version: 0.1.0 title: Api Documentation termsOfService: urn:tos contact: {} license: name: Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0 tags: - name: /connect servers: - url: /localhost paths: /connectors: get: tags: - KafkaConnectClient summary: get all connectors from Kafka Connect service operationId: getConnectors parameters: - name: search in: query required: false schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: type: string post: tags: - KafkaConnectClient summary: create new connector operationId: createConnector requestBody: content: application/json: schema: $ref: '#/components/schemas/NewConnector' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' 400: description: Bad request 409: description: rebalance is in progress 500: description: Internal server error /connectors/{connectorName}: get: tags: - KafkaConnectClient summary: get information about the connector operationId: getConnector parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' delete: tags: - KafkaConnectClient summary: delete connector operationId: deleteConnector parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK 409: description: rebalance is in progress /connectors/{connectorName}/config: get: tags: - KafkaConnectClient summary: get connector configuration operationId: getConnectorConfig parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' put: tags: - KafkaConnectClient summary: update or create connector with provided config operationId: setConnectorConfig parameters: - name: connectorName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' 400: description: Bad request 409: description: rebalance is in progress 500: description: Internal server error /connectors/{connectorName}/status: get: tags: - KafkaConnectClient summary: get connector status operationId: getConnectorStatus parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConnectorStatus' /connectors/{connectorName}/restart: post: tags: - KafkaConnectClient summary: restart the connector and its tasks operationId: restartConnector parameters: - name: connectorName in: path required: true schema: type: string - name: includeTasks in: query required: false schema: type: boolean default: false description: Specifies whether to restart the connector instance and task instances or just the connector instance - name: onlyFailed in: query required: false schema: type: boolean default: false description: Specifies whether to restart just the instances with a FAILED status or all instances responses: 200: description: OK 409: description: rebalance is in progress /connectors/{connectorName}/pause: put: tags: - KafkaConnectClient summary: pause the connector operationId: pauseConnector parameters: - name: connectorName in: path required: true schema: type: string responses: 202: description: Accepted /connectors/{connectorName}/resume: put: tags: - KafkaConnectClient summary: resume the connector operationId: resumeConnector parameters: - name: connectorName in: path required: true schema: type: string responses: 202: description: Accepted /connectors/{connectorName}/tasks: get: tags: - KafkaConnectClient summary: get connector tasks operationId: getConnectorTasks parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/ConnectorTask' /connectors/{connectorName}/topics: get: tags: - KafkaConnectClient summary: The set of topic names the connector has been using since its creation or since the last time its set of active topics was reset operationId: getConnectorTopics parameters: - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: object additionalProperties: $ref: '#/components/schemas/ConnectorTopics' /connectors/{connectorName}/tasks/{taskId}/status: get: tags: - KafkaConnectClient summary: get connector task status operationId: getConnectorTaskStatus parameters: - name: connectorName in: path required: true schema: type: string - name: taskId in: path required: true schema: type: integer responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/TaskStatus' /connectors/{connectorName}/tasks/{taskId}/restart: post: tags: - KafkaConnectClient summary: restart connector task operationId: restartConnectorTask parameters: - name: connectorName in: path required: true schema: type: string - name: taskId in: path required: true schema: type: integer responses: 200: description: OK /connector-plugins: get: tags: - KafkaConnectClient summary: get connector plugins operationId: getConnectorPlugins responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/ConnectorPlugin' /connector-plugins/{pluginName}/config/validate: put: tags: - KafkaConnectClient summary: validate connector plugin configuration operationId: validateConnectorPluginConfig parameters: - name: pluginName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConnectorPluginConfigValidationResponse' components: securitySchemes: basicAuth: type: http scheme: basic schemas: ConnectorConfig: type: object additionalProperties: type: object Task: type: object properties: connector: type: string task: type: integer ConnectorTask: type: object properties: id: $ref: '#/components/schemas/Task' config: $ref: '#/components/schemas/ConnectorConfig' NewConnector: type: object properties: name: type: string config: $ref: '#/components/schemas/ConnectorConfig' required: - name - config Connector: allOf: - $ref: '#/components/schemas/NewConnector' - type: object properties: tasks: type: array items: $ref: '#/components/schemas/Task' type: type: string enum: - source - sink TaskStatus: type: object properties: id: type: integer state: type: string enum: - RUNNING - FAILED - PAUSED - RESTARTING - UNASSIGNED worker_id: type: string trace: type: string ConnectorStatus: type: object properties: name: type: string connector: type: object properties: state: type: string enum: - RUNNING - FAILED - PAUSED - UNASSIGNED worker_id: type: string trace: type: string tasks: type: array items: $ref: '#/components/schemas/TaskStatus' ConnectorPlugin: type: object properties: class: type: string ConnectorPluginConfigDefinition: type: object properties: name: type: string type: type: string enum: - BOOLEAN - CLASS - DOUBLE - INT - LIST - LONG - PASSWORD - SHORT - STRING required: type: boolean default_value: type: string importance: type: string enum: - LOW - MEDIUM - HIGH documentation: type: string group: type: string width: type: string enum: - SHORT - MEDIUM - LONG - NONE display_name: type: string dependents: type: array items: type: string order: type: integer ConnectorPluginConfigValue: type: object properties: name: type: string value: type: string recommended_values: type: array items: type: string errors: type: array items: type: string visible: type: boolean ConnectorPluginConfig: type: object properties: definition: $ref: '#/components/schemas/ConnectorPluginConfigDefinition' value: $ref: '#/components/schemas/ConnectorPluginConfigValue' ConnectorPluginConfigValidationResponse: type: object properties: name: type: string error_count: type: integer groups: type: array items: type: string configs: type: array items: $ref: '#/components/schemas/ConnectorPluginConfig' ConnectorTopics: type: object properties: topics: type: array items: type: string security: - basicAuth: [] ================================================ FILE: kafka-ui-contract/src/main/resources/swagger/kafka-sr-api.yaml ================================================ openapi: 3.0.0 info: description: Api Documentation version: 0.1.0 title: Api Documentation termsOfService: urn:tos contact: {} license: name: Apache 2.0 url: http://www.apache.org/licenses/LICENSE-2.0 tags: - name: /schemaregistry servers: - url: /localhost paths: /subjects: get: tags: - KafkaSrClient summary: get all connectors from Kafka Connect service operationId: getAllSubjectNames parameters: - name: subjectPrefix in: query required: false schema: type: string - name: deleted in: query schema: type: boolean responses: 200: description: OK content: application/json: schema: #workaround for https://github.com/spring-projects/spring-framework/issues/24734 type: string /subjects/{subject}: delete: tags: - KafkaSrClient operationId: deleteAllSubjectVersions parameters: - name: subject in: path required: true schema: type: string - name: permanent in: query schema: type: boolean required: false responses: 200: description: OK 404: description: Not found /subjects/{subject}/versions/{version}: get: tags: - KafkaSrClient operationId: getSubjectVersion parameters: - name: subject in: path required: true schema: type: string - name: version in: path required: true schema: type: string - name: deleted in: query schema: type: boolean responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SchemaSubject' 404: description: Not found 422: description: Invalid version delete: tags: - KafkaSrClient operationId: deleteSubjectVersion parameters: - name: subject in: path required: true schema: type: string - name: permanent in: query required: false schema: type: boolean default: false - name: version in: path required: true schema: type: string responses: 200: description: OK 404: description: Not found /subjects/{subject}/versions: get: tags: - KafkaSrClient operationId: getSubjectVersions parameters: - name: subject in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: type: integer format: int32 404: description: Not found post: tags: - KafkaSrClient operationId: registerNewSchema parameters: - name: subject in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/NewSubject' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SubjectId' /config/: get: tags: - KafkaSrClient operationId: getGlobalCompatibilityLevel responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityConfig' 404: description: Not found put: tags: - KafkaSrClient operationId: updateGlobalCompatibilityLevel requestBody: content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevelChange' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevelChange' 404: description: Not found /config/{subject}: get: tags: - KafkaSrClient operationId: getSubjectCompatibilityLevel parameters: - name: subject in: path required: true schema: type: string - name: defaultToGlobal in: query required: true schema: type: boolean responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityConfig' 404: description: Not found put: tags: - KafkaSrClient operationId: updateSubjectCompatibilityLevel parameters: - name: subject in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevelChange' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevelChange' 404: description: Not found delete: tags: - KafkaSrClient operationId: deleteSubjectCompatibilityLevel parameters: - name: subject in: path required: true schema: type: string responses: 200: description: OK 404: description: Not found /compatibility/subjects/{subject}/versions/{version}: post: tags: - KafkaSrClient operationId: checkSchemaCompatibility parameters: - name: subject in: path required: true schema: type: string - name: version in: path required: true schema: type: string - name: verbose in: query description: Show reason a schema fails the compatibility test schema: type: boolean requestBody: content: application/json: schema: $ref: '#/components/schemas/NewSubject' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityCheckResponse' 404: description: Not found security: - basicAuth: [] components: securitySchemes: basicAuth: type: http scheme: basic schemas: SchemaSubject: type: object properties: subject: type: string version: type: string id: type: integer schema: type: string schemaType: $ref: '#/components/schemas/SchemaType' references: type: array items: $ref: '#/components/schemas/SchemaReference' required: - id - subject - version - schema - schemaType SchemaType: type: string description: upon updating a schema, the type of an existing schema can't be changed enum: - AVRO - JSON - PROTOBUF SchemaReference: type: object properties: name: type: string subject: type: string version: type: integer required: - name - subject - version SubjectId: type: object properties: id: type: integer NewSubject: type: object description: should be set for creating/updating schema subject properties: schema: type: string schemaType: $ref: '#/components/schemas/SchemaType' references: type: array items: $ref: '#/components/schemas/SchemaReference' required: - schema - schemaType CompatibilityConfig: type: object properties: compatibilityLevel: $ref: '#/components/schemas/Compatibility' required: - compatibilityLevel CompatibilityLevelChange: type: object properties: compatibility: $ref: '#/components/schemas/Compatibility' required: - compatibility Compatibility: type: string enum: - BACKWARD - BACKWARD_TRANSITIVE - FORWARD - FORWARD_TRANSITIVE - FULL - FULL_TRANSITIVE - NONE CompatibilityCheckResponse: type: object properties: is_compatible: type: boolean ================================================ FILE: kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml ================================================ openapi: 3.0.0 info: description: Api Documentation version: 0.1.0 title: Api Documentation termsOfService: urn:tos contact: { } license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0 tags: - name: /api/clusters - name: /api/clusters/connects servers: - url: /localhost paths: /api/clusters: get: tags: - Clusters summary: getClusters operationId: getClusters responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/Cluster' /api/clusters/{clusterName}/cache: post: tags: - Clusters summary: updateClusterInfo operationId: updateClusterInfo parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Cluster' 404: description: Not found /api/clusters/{clusterName}/brokers: get: tags: - Brokers summary: getBrokers operationId: getBrokers parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/Broker' /api/clusters/{clusterName}/brokers/{id}/configs: get: tags: - Brokers summary: getBrokerConfig operationId: getBrokerConfig parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: integer responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/BrokerConfig' 404: description: Not found /api/clusters/{clusterName}/brokers/{id}/configs/{name}: put: tags: - Brokers summary: updateBrokerConfigByName operationId: updateBrokerConfigByName parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: integer - name: name in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/BrokerConfigItem' responses: 200: description: OK /api/clusters/{clusterName}/metrics: get: tags: - Clusters summary: getClusterMetrics operationId: getClusterMetrics parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ClusterMetrics' /api/clusters/{clusterName}/stats: get: tags: - Clusters summary: getClusterStats operationId: getClusterStats parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ClusterStats' /api/clusters/{clusterName}/brokers/{id}/metrics: get: tags: - Brokers summary: getBrokersMetrics operationId: getBrokersMetrics parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: integer responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/BrokerMetrics' /api/clusters/{clusterName}/brokers/logdirs: get: tags: - Brokers summary: getAllBrokersLogdirs operationId: getAllBrokersLogdirs parameters: - name: clusterName in: path required: true schema: type: string - name: broker in: query description: array of broker ids required: false schema: type: array items: type: integer responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/BrokersLogdirs' /api/clusters/{clusterName}/brokers/{id}/logdirs: patch: tags: - Brokers summary: updateBrokerTopicPartitionLogDir operationId: updateBrokerTopicPartitionLogDir parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: integer requestBody: content: application/json: schema: $ref: '#/components/schemas/BrokerLogdirUpdate' responses: 200: description: OK /api/clusters/{clusterName}/topics: get: tags: - Topics summary: getTopics operationId: getTopics parameters: - name: clusterName in: path required: true schema: type: string - name: page in: query required: false schema: type: integer - name: perPage in: query required: false schema: type: integer - name: showInternal in: query required: false schema: type: boolean - name: search in: query required: false schema: type: string - name: orderBy in: query required: false schema: $ref: '#/components/schemas/TopicColumnsToSort' - name: sortOrder in: query required: false schema: $ref: '#/components/schemas/SortOrder' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/TopicsResponse' post: tags: - Topics summary: createTopic operationId: createTopic parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/TopicCreation' responses: 201: description: Created content: application/json: schema: $ref: '#/components/schemas/Topic' /api/clusters/{clusterName}/topics/{topicName}/clone: post: tags: - Topics summary: cloneTopic operationId: cloneTopic parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string - name: newTopicName in: query required: true schema: type: string responses: 201: description: Created content: application/json: schema: $ref: '#/components/schemas/Topic' 404: description: Not found /api/clusters/{clusterName}/topics/{topicName}/analysis: get: tags: - Topics summary: getTopicAnalysis operationId: getTopicAnalysis parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/TopicAnalysis' 404: description: Not found post: tags: - Topics summary: analyzeTopic operationId: analyzeTopic parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: Analysis started 404: description: Not found delete: tags: - Topics summary: cancelTopicAnalysis operationId: cancelTopicAnalysis parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: Analysis cancelled 404: description: Not found /api/clusters/{clusterName}/topics/{topicName}: get: tags: - Topics summary: getTopicDetails operationId: getTopicDetails parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/TopicDetails' post: tags: - Topics summary: recreateTopic operationId: recreateTopic parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 201: description: Created content: application/json: schema: $ref: '#/components/schemas/Topic' 404: description: Not found 408: description: Topic recreation timeout patch: tags: - Topics summary: updateTopic operationId: updateTopic parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/TopicUpdate' responses: 200: description: Updated content: application/json: schema: $ref: '#/components/schemas/Topic' delete: tags: - Topics summary: deleteTopic operationId: deleteTopic parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK 404: description: Not found /api/clusters/{clusterName}/topics/{topicName}/config: get: tags: - Topics summary: getTopicConfigs operationId: getTopicConfigs parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/TopicConfig' /api/clusters/{clusterName}/topics/{topicName}/replications: patch: tags: - Topics summary: changeReplicationFactor operationId: changeReplicationFactor parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ReplicationFactorChange' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ReplicationFactorChangeResponse' 404: description: Not found 400: description: Bad Request /api/clusters/{clusterName}/topic/{topicName}/serdes: get: tags: - Messages summary: getSerdes operationId: getSerdes parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string - name: use in: query required: true schema: $ref: '#/components/schemas/SerdeUsage' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/TopicSerdeSuggestion' /api/smartfilters/testexecutions: put: tags: - Messages summary: executeSmartFilterTest operationId: executeSmartFilterTest requestBody: content: application/json: schema: $ref: '#/components/schemas/SmartFilterTestExecution' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SmartFilterTestExecutionResult' /api/clusters/{clusterName}/topics/{topicName}/messages: get: tags: - Messages summary: getTopicMessages operationId: getTopicMessages parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string - name: seekType in: query schema: $ref: "#/components/schemas/SeekType" - name: seekTo in: query schema: type: array items: type: string description: The format is [partition]::[offset] for specifying offsets or [partition]::[timestamp in millis] for specifying timestamps - name: limit in: query schema: type: integer - name: q in: query schema: type: string - name: filterQueryType in: query schema: $ref: "#/components/schemas/MessageFilterType" - name: seekDirection in: query schema: $ref: "#/components/schemas/SeekDirection" - name: keySerde in: query description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string - name: valueSerde in: query description: "Serde that should be used for deserialization. Will be chosen automatically if not set." schema: type: string responses: 200: description: OK content: text/event-stream: schema: type: array items: $ref: '#/components/schemas/TopicMessageEvent' delete: tags: - Messages summary: deleteTopicMessages operationId: deleteTopicMessages parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string - name: partitions in: query required: false schema: type: array items: type: integer responses: 200: description: OK 404: description: Not found post: tags: - Messages summary: sendTopicMessages operationId: sendTopicMessages parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CreateTopicMessage' responses: 200: description: OK 404: description: Not found /api/clusters/{clusterName}/topics/{topicName}/activeproducers: get: tags: - Topics summary: get producer states for topic operationId: getActiveProducerStates parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/TopicProducerState' /api/clusters/{clusterName}/topics/{topicName}/consumer-groups: get: tags: - Consumer Groups summary: get Consumer Groups By Topics operationId: getTopicConsumerGroups parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/ConsumerGroup' /api/clusters/{clusterName}/consumer-groups/paged: get: tags: - Consumer Groups summary: Get consumer groups with paging support operationId: getConsumerGroupsPage parameters: - name: clusterName in: path required: true schema: type: string - name: page in: query required: false schema: type: integer - name: perPage in: query required: false schema: type: integer - name: search in: query required: false schema: type: string - name: orderBy in: query required: false schema: $ref: '#/components/schemas/ConsumerGroupOrdering' - name: sortOrder in: query required: false schema: $ref: '#/components/schemas/SortOrder' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConsumerGroupsPageResponse' /api/clusters/{clusterName}/consumer-groups/{id}: get: tags: - Consumer Groups summary: get Consumer Group By Id operationId: getConsumerGroup parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConsumerGroupDetails' delete: tags: - Consumer Groups summary: Delete Consumer Group by ID operationId: deleteConsumerGroup parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: string responses: 200: description: OK /api/clusters/{clusterName}/consumer-groups/{id}/offsets: post: tags: - Consumer Groups summary: resets consumer group offsets operationId: resetConsumerGroupOffsets parameters: - name: clusterName in: path required: true schema: type: string - name: id in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ConsumerGroupOffsetsReset' responses: 200: description: OK /api/clusters/{clusterName}/schemas: post: tags: - Schemas summary: create a new subject schema or update existing subject schema operationId: createNewSchema parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/NewSchemaSubject' responses: 200: description: Ok content: application/json: schema: $ref: '#/components/schemas/SchemaSubject' 400: description: Bad request 409: description: Duplicate schema 422: description: Invalid parameters get: tags: - Schemas summary: get all schemas of latest version from Schema Registry service operationId: getSchemas parameters: - name: clusterName in: path required: true schema: type: string - name: page in: query required: false schema: type: integer - name: perPage in: query required: false schema: type: integer - name: search in: query required: false schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SchemaSubjectsResponse' /api/clusters/{clusterName}/schemas/{subject}: delete: tags: - Schemas summary: delete schema from Schema Registry service operationId: deleteSchema parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string responses: 200: description: OK 404: description: Not found /api/clusters/{clusterName}/schemas/{subject}/versions: get: tags: - Schemas summary: get all version of subject from Schema Registry service operationId: getAllVersionsBySubject parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/SchemaSubject' /api/clusters/{clusterName}/schemas/{subject}/latest: get: tags: - Schemas summary: get the latest schema from Schema Registry service operationId: getLatestSchema parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SchemaSubject' delete: tags: - Schemas summary: delete the latest schema from schema registry operationId: deleteLatestSchema parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string responses: 200: description: OK 404: description: Not found /api/clusters/{clusterName}/schemas/{subject}/versions/{version}: get: tags: - Schemas summary: get schema by version from Schema Registry service operationId: getSchemaByVersion parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string - name: version in: path required: true schema: type: integer responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/SchemaSubject' delete: tags: - Schemas summary: delete schema by version from schema registry operationId: deleteSchemaByVersion parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string - name: version in: path required: true schema: type: integer responses: 200: description: OK 404: description: Not found /api/clusters/{clusterName}/schemas/compatibility: get: tags: - Schemas summary: Get global schema compatibility level operationId: getGlobalSchemaCompatibilityLevel parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevel' put: tags: - Schemas summary: Update compatibility level globally operationId: updateGlobalSchemaCompatibilityLevel parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevel' responses: 200: description: OK 404: description: Not Found /api/clusters/{clusterName}/schemas/{subject}/compatibility: put: tags: - Schemas summary: Update compatibility level for specific schema. operationId: updateSchemaCompatibilityLevel parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CompatibilityLevel' responses: 200: description: OK 404: description: Not Found /api/clusters/{clusterName}/schemas/{subject}/check: post: tags: - Schemas summary: Check compatibility of the schema. operationId: checkSchemaCompatibility parameters: - name: clusterName in: path required: true schema: type: string - name: subject in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/NewSchemaSubject' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/CompatibilityCheckResponse' 404: description: Not Found /api/clusters/{clusterName}/connects: get: tags: - Kafka Connect summary: get all kafka connect instances operationId: getConnects parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/Connect' /api/clusters/{clusterName}/connectors: get: tags: - Kafka Connect summary: get filtered kafka connectors operationId: getAllConnectors parameters: - name: clusterName in: path required: true schema: type: string - name: search in: query required: false schema: type: string - name: orderBy in: query required: false schema: $ref: '#/components/schemas/ConnectorColumnsToSort' - name: sortOrder in: query required: false schema: $ref: '#/components/schemas/SortOrder' responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/FullConnectorInfo' /api/clusters/{clusterName}/connects/{connectName}/connectors: get: tags: - Kafka Connect summary: get connectors for provided kafka connect instance operationId: getConnectors parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: type: string post: tags: - Kafka Connect summary: create new connector operationId: createConnector parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/NewConnector' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' 409: description: rebalance is in progress /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}: get: tags: - Kafka Connect summary: get information about the connector operationId: getConnector parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' delete: tags: - Kafka Connect summary: delete connector operationId: deleteConnector parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string responses: 200: description: OK 409: description: rebalance is in progress /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/action/{action}: post: tags: - Kafka Connect summary: update connector state (restart, pause or resume) operationId: updateConnectorState parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string - name: action in: path required: true schema: $ref: '#/components/schemas/ConnectorAction' responses: 200: description: OK 409: description: rebalance is in progress /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/config: get: tags: - Kafka Connect summary: get connector configuration operationId: getConnectorConfig parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' put: tags: - Kafka Connect summary: update or create connector with provided config operationId: setConnectorConfig parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/Connector' 409: description: rebalance is in progress /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/tasks: get: tags: - Kafka Connect summary: get connector tasks operationId: getConnectorTasks parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/Task' /api/clusters/{clusterName}/connects/{connectName}/connectors/{connectorName}/tasks/{taskId}/action/restart: post: tags: - Kafka Connect summary: restart connector task operationId: restartConnectorTask parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: connectorName in: path required: true schema: type: string - name: taskId in: path required: true schema: type: integer responses: 200: description: OK /api/clusters/{clusterName}/ksql/v2: post: tags: - Ksql summary: executeKsql operationId: executeKsql parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/KsqlCommandV2' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/KsqlCommandV2Response' /api/clusters/{clusterName}/ksql/tables: get: tags: - Ksql summary: listTables operationId: listTables parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/KsqlTableDescription' /api/clusters/{clusterName}/ksql/streams: get: tags: - Ksql summary: listStreams operationId: listStreams parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/KsqlStreamDescription' /api/clusters/{clusterName}/ksql/response: get: tags: - Ksql summary: Open SSE pipe operationId: openKsqlResponsePipe parameters: - name: clusterName in: path required: true schema: type: string - name: pipeId in: query required: true schema: type: string responses: 200: description: OK content: text/event-stream: schema: type: array items: $ref: '#/components/schemas/KsqlResponse' /api/clusters/{clusterName}/connects/{connectName}/plugins: get: tags: - Kafka Connect summary: get connector plugins operationId: getConnectorPlugins parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/ConnectorPlugin' /api/clusters/{clusterName}/connects/{connectName}/plugins/{pluginName}/config/validate: put: tags: - Kafka Connect summary: validate connector plugin configuration operationId: validateConnectorPluginConfig parameters: - name: clusterName in: path required: true schema: type: string - name: connectName in: path required: true schema: type: string - name: pluginName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/ConnectorConfig' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ConnectorPluginConfigValidationResponse' /api/clusters/{clusterName}/topics/{topicName}/partitions: patch: tags: - Topics summary: increaseTopicPartitions operationId: increaseTopicPartitions parameters: - name: clusterName in: path required: true schema: type: string - name: topicName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/PartitionsIncrease' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/PartitionsIncreaseResponse' 404: description: Not found /api/clusters/{clusterName}/acls: get: tags: - Acls summary: listKafkaAcls operationId: listAcls parameters: - name: clusterName in: path required: true schema: type: string - name: resourceType in: query required: false schema: $ref: '#/components/schemas/KafkaAclResourceType' - name: resourceName in: query required: false schema: type: string - name: namePatternType in: query required: false schema: $ref: '#/components/schemas/KafkaAclNamePatternType' responses: 200: description: OK content: application/json: schema: type: array items: $ref: '#/components/schemas/KafkaAcl' /api/clusters/{clusterName}/acl/csv: get: tags: - Acls summary: getAclAsCsv operationId: getAclAsCsv parameters: - name: clusterName in: path required: true schema: type: string responses: 200: description: OK content: text/plain: schema: type: string post: tags: - Acls summary: syncAclsCsv operationId: syncAclsCsv parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: text/plain: schema: type: string responses: 200: description: OK /api/clusters/{clusterName}/acl: post: tags: - Acls summary: createAcl operationId: createAcl parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/KafkaAcl' responses: 200: description: OK delete: tags: - Acls summary: deleteAcl operationId: deleteAcl parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/KafkaAcl' responses: 200: description: OK 404: description: Acl not found /api/clusters/{clusterName}/acl/consumer: post: tags: - Acls summary: createConsumerAcl operationId: createConsumerAcl parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CreateConsumerAcl' responses: 200: description: OK /api/clusters/{clusterName}/acl/producer: post: tags: - Acls summary: createProducerAcl operationId: createProducerAcl parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CreateProducerAcl' responses: 200: description: OK /api/clusters/{clusterName}/acl/streamApp: post: tags: - Acls summary: createStreamAppAcl operationId: createStreamAppAcl parameters: - name: clusterName in: path required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/CreateStreamAppAcl' responses: 200: description: OK /api/authorization: get: tags: - Authorization summary: Get user authentication related info operationId: getUserAuthInfo responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/AuthenticationInfo' /api/info: get: tags: - ApplicationConfig summary: Gets application info operationId: getApplicationInfo responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ApplicationInfo' /api/config: get: tags: - ApplicationConfig summary: Gets current application configuration operationId: getCurrentConfig responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ApplicationConfig' put: tags: - ApplicationConfig summary: Restarts application with specified configuration operationId: restartWithConfig requestBody: content: application/json: schema: $ref: '#/components/schemas/RestartRequest' responses: 200: description: OK /api/config/validated: put: tags: - ApplicationConfig summary: Restarts application with specified configuration operationId: validateConfig requestBody: content: application/json: schema: $ref: '#/components/schemas/ApplicationConfig' responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/ApplicationConfigValidation' /api/config/relatedfiles: post: tags: - ApplicationConfig summary: Restarts application with specified configuration operationId: uploadConfigRelatedFile requestBody: content: multipart/form-data: schema: type: object properties: file: type: string format: binary responses: 200: description: OK content: application/json: schema: $ref: '#/components/schemas/UploadedFileInfo' components: schemas: TopicSerdeSuggestion: type: object properties: key: type: array items: $ref: '#/components/schemas/SerdeDescription' value: type: array items: $ref: '#/components/schemas/SerdeDescription' SerdeDescription: type: object properties: name: type: string description: type: string preferred: description: "This serde was automatically chosen by cluster config. This should be enabled in UI by default. Also it will be used for deserialization if no serdes passed." type: boolean schema: type: string additionalProperties: type: object additionalProperties: type: object SerdeUsage: type: string enum: - SERIALIZE - DESERIALIZE ErrorResponse: description: Error object that will be returned with 4XX and 5XX HTTP statuses type: object properties: code: type: integer description: Internal error code (can be used for message formatting & localization on UI) message: type: string description: Error message timestamp: type: number description: Response unix timestamp in ms requestId: type: string description: Unique server-defined request id for convenient debugging fieldsErrors: type: array items: $ref: '#/components/schemas/FieldError' stackTrace: type: string FieldError: type: object properties: fieldName: type: string description: Name of field that violated format restrictions: description: Field format violations description (ex. ["size must be between 0 and 20", "must be a well-formed email address"]) type: array items: type: string MetricsCollectionError: type: object properties: message: type: string stackTrace: type: string ApplicationInfo: type: object properties: enabledFeatures: type: array items: type: string enum: - DYNAMIC_CONFIG build: type: object properties: commitId: type: string version: type: string buildTime: type: string isLatestRelease: type: boolean latestRelease: type: object properties: versionTag: type: string publishedAt: type: string htmlUrl: type: string Cluster: type: object properties: name: type: string defaultCluster: type: boolean status: $ref: '#/components/schemas/ServerStatus' lastError: $ref: '#/components/schemas/MetricsCollectionError' brokerCount: type: integer onlinePartitionCount: type: integer topicCount: type: integer bytesInPerSec: type: number bytesOutPerSec: type: number readOnly: type: boolean version: type: string features: type: array items: type: string enum: - SCHEMA_REGISTRY - KAFKA_CONNECT - KSQL_DB - TOPIC_DELETION - KAFKA_ACL_VIEW # get ACLs listing - KAFKA_ACL_EDIT # create & delete ACLs required: - id - name - status ServerStatus: type: string enum: - online - offline - initializing ClusterMetrics: type: object properties: items: type: array items: $ref: '#/components/schemas/Metric' ClusterStats: type: object properties: brokerCount: type: integer zooKeeperStatus: type: integer deprecated: true activeControllers: type: integer description: Id of broker which is cluster's controller. null, if controller not known yet. onlinePartitionCount: type: integer offlinePartitionCount: type: integer inSyncReplicasCount: type: integer outOfSyncReplicasCount: type: integer underReplicatedPartitionCount: type: integer diskUsage: type: array items: $ref: '#/components/schemas/BrokerDiskUsage' version: type: string BrokerDiskUsage: type: object properties: brokerId: type: integer segmentSize: type: integer format: int64 segmentCount: type: integer required: - brokerId BrokerMetrics: type: object properties: segmentSize: type: integer format: int64 segmentCount: type: integer metrics: type: array items: $ref: '#/components/schemas/Metric' BrokerLogdirs: type: object properties: name: type: string error: type: string topics: type: array items: $ref: '#/components/schemas/TopicLogdirs' BrokersLogdirs: type: object properties: name: type: string error: type: string topics: type: array items: $ref: '#/components/schemas/BrokerTopicLogdirs' TopicsResponse: type: object properties: pageCount: type: integer topics: type: array items: $ref: '#/components/schemas/Topic' TopicColumnsToSort: type: string enum: - NAME - OUT_OF_SYNC_REPLICAS - TOTAL_PARTITIONS - REPLICATION_FACTOR - SIZE ConnectorColumnsToSort: type: string enum: - NAME - CONNECT - TYPE - STATUS SortOrder: type: string enum: - ASC - DESC Topic: type: object properties: name: type: string internal: type: boolean partitionCount: type: integer replicationFactor: type: integer replicas: type: integer inSyncReplicas: type: integer segmentSize: type: integer format: int64 segmentCount: type: integer bytesInPerSec: type: number bytesOutPerSec: type: number underReplicatedPartitions: type: integer cleanUpPolicy: $ref: '#/components/schemas/CleanUpPolicy' partitions: type: array items: $ref: "#/components/schemas/Partition" required: - name TopicAnalysis: type: object description: "Represents analysis state. Note: 'progress' and 'result' fields are set exclusively depending on analysis state." properties: progress: $ref: '#/components/schemas/TopicAnalysisProgress' result: $ref: '#/components/schemas/TopicAnalysisResult' TopicAnalysisProgress: type: object properties: startedAt: type: integer format: int64 completenessPercent: type: number msgsScanned: type: integer format: int64 bytesScanned: type: integer format: int64 TopicAnalysisResult: type: object properties: startedAt: type: integer format: int64 finishedAt: type: integer format: int64 error: type: string totalStats: $ref: '#/components/schemas/TopicAnalysisStats' partitionStats: type: array items: $ref: "#/components/schemas/TopicAnalysisStats" TopicAnalysisStats: type: object properties: partition: type: integer format: int32 description: "null if this is total stats" totalMsgs: type: integer format: int64 minOffset: type: integer format: int64 maxOffset: type: integer format: int64 minTimestamp: type: integer format: int64 maxTimestamp: type: integer format: int64 nullKeys: type: integer format: int64 nullValues: type: integer format: int64 approxUniqKeys: type: integer format: int64 approxUniqValues: type: integer format: int64 keySize: $ref: "#/components/schemas/TopicAnalysisSizeStats" valueSize: $ref: "#/components/schemas/TopicAnalysisSizeStats" hourlyMsgCounts: type: array items: type: object properties: hourStart: type: integer format: int64 count: type: integer format: int64 TopicAnalysisSizeStats: type: object description: "All sizes in bytes" properties: sum: type: integer format: int64 min: type: integer format: int64 max: type: integer format: int64 avg: type: integer format: int64 prctl50: type: integer format: int64 prctl75: type: integer format: int64 prctl95: type: integer format: int64 prctl99: type: integer format: int64 prctl999: type: integer format: int64 Replica: type: object properties: broker: type: integer leader: type: boolean inSync: type: boolean TopicDetails: type: object properties: name: type: string internal: type: boolean partitions: type: array items: $ref: "#/components/schemas/Partition" partitionCount: type: integer replicationFactor: type: integer replicas: type: integer inSyncReplicas: type: integer bytesInPerSec: type: number bytesOutPerSec: type: number segmentSize: type: integer format: int64 segmentCount: type: integer underReplicatedPartitions: type: integer cleanUpPolicy: $ref: '#/components/schemas/CleanUpPolicy' keySerde: type: string valueSerde: type: string required: - name TopicConfig: type: object properties: name: type: string value: type: string defaultValue: type: string source: $ref: "#/components/schemas/ConfigSource" isSensitive: type: boolean isReadOnly: type: boolean synonyms: type: array items: $ref: "#/components/schemas/ConfigSynonym" doc: type: string required: - name TopicCreation: type: object properties: name: type: string partitions: type: integer replicationFactor: type: integer configs: type: object additionalProperties: type: string required: - name - partitions TopicUpdate: type: object properties: configs: type: object additionalProperties: type: string required: - configs Broker: type: object properties: id: type: integer host: type: string port: type: integer bytesInPerSec: type: number bytesOutPerSec: type: number partitionsLeader: type: integer partitions: type: integer inSyncPartitions: type: integer partitionsSkew: type: number leadersSkew: type: number required: - id BrokerLogdirUpdate: type: object properties: topic: type: string partition: type: integer logDir: type: string ConsumerGroupState: type: string enum: - UNKNOWN - PREPARING_REBALANCE - COMPLETING_REBALANCE - STABLE - DEAD - EMPTY MessageFormat: type: string enum: - AVRO - JSON - PROTOBUF - UNKNOWN TopicProducerState: type: object properties: partition: type: integer format: int32 producerId: type: integer format: int64 producerEpoch: type: integer format: int32 lastSequence: type: integer format: int32 lastTimestampMs: type: integer format: int64 coordinatorEpoch: type: integer format: int32 currentTransactionStartOffset: type: integer format: int64 ConsumerGroup: discriminator: propertyName: inherit mapping: details: "#/components/schemas/ConsumerGroupDetails" type: object properties: groupId: type: string members: type: integer topics: type: integer simple: type: boolean partitionAssignor: type: string state: $ref: "#/components/schemas/ConsumerGroupState" coordinator: $ref: "#/components/schemas/Broker" consumerLag: type: integer format: int64 description: null if consumer group has no offsets committed required: - groupId ConsumerGroupOrdering: type: string enum: - NAME - MEMBERS - STATE - MESSAGES_BEHIND - TOPIC_NUM ConsumerGroupsPageResponse: type: object properties: pageCount: type: integer consumerGroups: type: array items: $ref: '#/components/schemas/ConsumerGroup' SmartFilterTestExecution: type: object required: [filterCode] properties: filterCode: type: string key: type: string value: type: string headers: type: object additionalProperties: type: string partition: type: integer offset: type: integer format: int64 timestampMs: type: integer format: int64 SmartFilterTestExecutionResult: type: object properties: result: type: boolean error: type: string CreateTopicMessage: type: object properties: partition: type: integer key: type: string nullable: true headers: type: object additionalProperties: type: string content: type: string nullable: true keySerde: type: string nullable: true valueSerde: type: string nullable: true required: - partition TopicMessageEvent: type: object properties: type: type: string enum: - PHASE - MESSAGE - CONSUMING - DONE - EMIT_THROTTLING message: $ref: "#/components/schemas/TopicMessage" phase: $ref: "#/components/schemas/TopicMessagePhase" consuming: $ref: "#/components/schemas/TopicMessageConsuming" TopicMessagePhase: type: object properties: name: type: string TimeStampFormat: type: object properties: timeStampFormat: type: string TopicMessageConsuming: type: object properties: bytesConsumed: type: integer format: int64 elapsedMs: type: integer format: int64 isCancelled: type: boolean messagesConsumed: type: integer filterApplyErrors: type: integer TopicMessage: type: object properties: partition: type: integer offset: type: integer format: int64 timestamp: type: string format: date-time timestampType: type: string enum: - NO_TIMESTAMP_TYPE - CREATE_TIME - LOG_APPEND_TIME key: type: string headers: type: object additionalProperties: type: string content: type: string keyFormat: #deprecated - wont be filled - use 'keySerde' field instead $ref: "#/components/schemas/MessageFormat" valueFormat: #deprecated - wont be filled - use 'valueSerde' field instead $ref: "#/components/schemas/MessageFormat" keySize: type: integer format: int64 valueSize: type: integer format: int64 keySchemaId: deprecated: true description: deprecated - wont be filled - use 'keyDeserializeProperties' field instead type: string valueSchemaId: deprecated: true description: deprecated - wont be filled - use 'valueDeserializeProperties' field instead type: string headersSize: type: integer format: int64 keySerde: type: string valueSerde: type: string keyDeserializeProperties: additionalProperties: type: object valueDeserializeProperties: additionalProperties: type: object required: - partition - offset - timestamp SeekType: type: string enum: - BEGINNING - OFFSET - TIMESTAMP - LATEST MessageFilterType: type: string enum: - STRING_CONTAINS - GROOVY_SCRIPT SeekDirection: type: string enum: - FORWARD - BACKWARD - TAILING default: FORWARD Partition: type: object properties: partition: type: integer leader: type: integer replicas: type: array items: $ref: '#/components/schemas/Replica' offsetMax: type: integer format: int64 offsetMin: type: integer format: int64 required: - topic - partition - offsetMax - offsetMin ConsumerGroupTopicPartition: type: object properties: topic: type: string partition: type: integer currentOffset: type: integer format: int64 endOffset: type: integer format: int64 consumerLag: type: integer format: int64 description: null if consumer group has no offsets committed consumerId: type: string host: type: string required: - topic - partition ConsumerGroupDetails: allOf: - $ref: '#/components/schemas/ConsumerGroup' - type: object properties: partitions: type: array items: $ref: '#/components/schemas/ConsumerGroupTopicPartition' Metric: type: object properties: name: type: string labels: type: string additionalProperties: type: string value: type: number TopicLogdirs: type: object properties: name: type: string partitions: type: array items: $ref: '#/components/schemas/TopicPartitionLogdir' BrokerTopicLogdirs: type: object properties: name: type: string partitions: type: array items: $ref: '#/components/schemas/BrokerTopicPartitionLogdir' TopicPartitionLogdir: type: object properties: partition: type: integer size: type: integer format: int64 offsetLag: type: integer format: int64 BrokerTopicPartitionLogdir: allOf: - $ref: '#/components/schemas/TopicPartitionLogdir' - type: object properties: broker: type: integer SchemaSubject: type: object properties: subject: type: string version: type: string id: type: integer schema: type: string compatibilityLevel: type: string schemaType: $ref: '#/components/schemas/SchemaType' references: type: array items: $ref: '#/components/schemas/SchemaReference' required: - id - subject - version - schema - compatibilityLevel - schemaType NewSchemaSubject: type: object description: should be set for creating/updating schema subject properties: subject: type: string schema: type: string schemaType: $ref: '#/components/schemas/SchemaType' # upon updating a schema, the type of existing schema can't be changed references: type: array items: $ref: '#/components/schemas/SchemaReference' required: - subject - schema - schemaType SchemaReference: type: object properties: name: type: string subject: type: string version: type: integer required: - name - subject - version CompatibilityLevel: type: object properties: compatibility: type: string enum: - BACKWARD - BACKWARD_TRANSITIVE - FORWARD - FORWARD_TRANSITIVE - FULL - FULL_TRANSITIVE - NONE required: - compatibility SchemaType: type: string description: upon updating a schema, the type of an existing schema can't be changed enum: - AVRO - JSON - PROTOBUF CompatibilityCheckResponse: type: object properties: isCompatible: type: boolean required: - isCompatible SchemaSubjectsResponse: type: object properties: pageCount: type: integer schemas: type: array items: $ref: '#/components/schemas/SchemaSubject' Connect: type: object properties: name: type: string address: type: string required: - name ConnectorConfig: type: object additionalProperties: type: object TaskId: type: object properties: connector: type: string task: type: integer Task: type: object properties: id: $ref: '#/components/schemas/TaskId' status: $ref: '#/components/schemas/TaskStatus' config: $ref: '#/components/schemas/ConnectorConfig' required: - status NewConnector: type: object properties: name: type: string config: $ref: '#/components/schemas/ConnectorConfig' required: - name - config Connector: allOf: - $ref: '#/components/schemas/NewConnector' - type: object properties: tasks: type: array items: $ref: '#/components/schemas/TaskId' type: $ref: '#/components/schemas/ConnectorType' status: $ref: '#/components/schemas/ConnectorStatus' connect: type: string required: - type - status - connect ConnectorType: type: string enum: - SOURCE - SINK ConsumerGroupOffsetsReset: type: object properties: topic: type: string resetType: $ref: '#/components/schemas/ConsumerGroupOffsetsResetType' partitions: type: array items: type: integer description: list of target partitions, all partitions will be used if it is not set or empty resetToTimestamp: type: integer format: int64 description: should be set if resetType is TIMESTAMP partitionsOffsets: type: array items: $ref: '#/components/schemas/PartitionOffset' description: List of partition offsets to reset to, should be set when resetType is OFFSET required: - topic - resetType PartitionOffset: type: object properties: partition: type: integer offset: type: integer format: int64 required: - partition ConsumerGroupOffsetsResetType: type: string enum: - EARLIEST - LATEST - TIMESTAMP - OFFSET TaskStatus: type: object properties: id: type: integer state: $ref: '#/components/schemas/ConnectorTaskStatus' worker_id: type: string trace: type: string required: - id - state - worker_id ConnectorStatus: type: object properties: state: $ref: '#/components/schemas/ConnectorState' worker_id: type: string required: - state ConnectorTaskStatus: type: string enum: - RUNNING - FAILED - PAUSED - RESTARTING - UNASSIGNED ConnectorState: type: string enum: - RUNNING - FAILED - PAUSED - UNASSIGNED - TASK_FAILED ConnectorAction: type: string enum: - RESTART - RESTART_ALL_TASKS - RESTART_FAILED_TASKS - PAUSE - RESUME TaskAction: type: string enum: - restart ConnectorPlugin: type: object properties: class: type: string ConnectorPluginConfigDefinition: type: object properties: name: type: string type: type: string enum: - BOOLEAN - CLASS - DOUBLE - INT - LIST - LONG - PASSWORD - SHORT - STRING required: type: boolean default_value: type: string importance: type: string enum: - LOW - MEDIUM - HIGH documentation: type: string group: type: string width: type: string enum: - SHORT - MEDIUM - LONG - NONE display_name: type: string dependents: type: array items: type: string order: type: integer ConnectorPluginConfigValue: type: object properties: name: type: string value: type: string recommended_values: type: array items: type: string errors: type: array items: type: string visible: type: boolean ConnectorPluginConfig: type: object properties: definition: $ref: '#/components/schemas/ConnectorPluginConfigDefinition' value: $ref: '#/components/schemas/ConnectorPluginConfigValue' ConnectorPluginConfigValidationResponse: type: object properties: name: type: string error_count: type: integer groups: type: array items: type: string configs: type: array items: $ref: '#/components/schemas/ConnectorPluginConfig' KsqlCommandV2: type: object properties: ksql: type: string streamsProperties: type: object additionalProperties: type: string required: - ksql KsqlCommandV2Response: type: object properties: pipeId: type: string required: - pipeId KsqlTableDescription: type: object properties: name: type: string topic: type: string keyFormat: type: string valueFormat: type: string isWindowed: type: boolean KsqlStreamDescription: type: object properties: name: type: string topic: type: string keyFormat: type: string valueFormat: type: string KsqlResponse: type: object properties: table: $ref: '#/components/schemas/KsqlTableResponse' KsqlTableResponse: type: object properties: header: type: string columnNames: type: array items: type: string values: type: array items: type: array items: type: object FullConnectorInfo: type: object properties: connect: type: string name: type: string connector_class: type: string type: $ref: '#/components/schemas/ConnectorType' topics: type: array items: type: string status: $ref: '#/components/schemas/ConnectorStatus' tasks_count: type: integer failed_tasks_count: type: integer required: - name - connect - status PartitionsIncrease: type: object properties: totalPartitionsCount: type: integer minimum: 1 required: - totalPartitionsCount PartitionsIncreaseResponse: type: object properties: totalPartitionsCount: type: integer topicName: type: string required: - totalPartitionsCount - topicName ReplicationFactorChange: type: object properties: totalReplicationFactor: type: integer required: - totalReplicationFactor ReplicationFactorChangeResponse: type: object properties: totalReplicationFactor: type: integer topicName: type: string required: - totalReplicationFactor - topicName BrokerConfigItem: type: object properties: value: type: string BrokerConfig: type: object properties: name: type: string value: type: string source: $ref: '#/components/schemas/ConfigSource' isSensitive: type: boolean isReadOnly: type: boolean synonyms: type: array items: $ref: '#/components/schemas/ConfigSynonym' required: - name - value - source - isSensitive - isReadOnly ConfigSource: type: string enum: - DYNAMIC_TOPIC_CONFIG - DYNAMIC_BROKER_LOGGER_CONFIG - DYNAMIC_BROKER_CONFIG - DYNAMIC_DEFAULT_BROKER_CONFIG - STATIC_BROKER_CONFIG - DEFAULT_CONFIG - UNKNOWN ConfigSynonym: type: object properties: name: type: string value: type: string source: $ref: '#/components/schemas/ConfigSource' CleanUpPolicy: type: string enum: - DELETE - COMPACT - COMPACT_DELETE - UNKNOWN AuthenticationInfo: type: object properties: rbacEnabled: type: boolean description: true if role based access control is enabled and granular permission access is required userInfo: $ref: '#/components/schemas/UserInfo' required: - rbacEnabled UserInfo: type: object properties: username: type: string permissions: type: array items: $ref: '#/components/schemas/UserPermission' required: - username - permissions UserPermission: type: object properties: clusters: type: array items: type: string resource: $ref: '#/components/schemas/ResourceType' value: type: string actions: type: array items: $ref: '#/components/schemas/Action' required: - clusters - resource - actions Action: type: string enum: - VIEW - EDIT - CREATE - DELETE - RESET_OFFSETS - EXECUTE - MODIFY_GLOBAL_COMPATIBILITY - ANALYSIS_VIEW - ANALYSIS_RUN - MESSAGES_READ - MESSAGES_PRODUCE - MESSAGES_DELETE - RESTART ResourceType: type: string enum: - APPLICATIONCONFIG - CLUSTERCONFIG - TOPIC - CONSUMER - SCHEMA - CONNECT - KSQL - ACL - AUDIT KafkaAcl: type: object required: [resourceType, resourceName, namePatternType, principal, host, operation, permission] properties: resourceType: $ref: '#/components/schemas/KafkaAclResourceType' resourceName: type: string # "*" if acl can be applied to any resource of given type namePatternType: $ref: '#/components/schemas/KafkaAclNamePatternType' principal: type: string host: type: string operation: type: string enum: - UNKNOWN # Unknown operation, need to update mapping code on BE - ALL # Cluster, Topic, Group - READ # Topic, Group - WRITE # Topic, TransactionalId - CREATE # Cluster, Topic - DELETE # Topic, Group - ALTER # Cluster, Topic, - DESCRIBE # Cluster, Topic, Group, TransactionalId, DelegationToken - CLUSTER_ACTION # Cluster - DESCRIBE_CONFIGS # Cluster, Topic - ALTER_CONFIGS # Cluster, Topic - IDEMPOTENT_WRITE # Cluster - CREATE_TOKENS - DESCRIBE_TOKENS permission: type: string enum: - ALLOW - DENY CreateConsumerAcl: type: object required: [principal, host] properties: principal: type: string host: type: string topics: type: array items: type: string topicsPrefix: type: string consumerGroups: type: array items: type: string consumerGroupsPrefix: type: string CreateProducerAcl: type: object required: [principal, host] properties: principal: type: string host: type: string topics: type: array items: type: string topicsPrefix: type: string transactionalId: type: string transactionsIdPrefix: type: string idempotent: type: boolean default: false CreateStreamAppAcl: type: object required: [principal, host, applicationId, inputTopics, outputTopics] properties: principal: type: string host: type: string inputTopics: type: array items: type: string outputTopics: type: array items: type: string applicationId: nullable: false type: string KafkaAclResourceType: type: string enum: - UNKNOWN # Unknown operation, need to update mapping code on BE - TOPIC - GROUP - CLUSTER - TRANSACTIONAL_ID - DELEGATION_TOKEN - USER KafkaAclNamePatternType: type: string enum: - MATCH - LITERAL - PREFIXED RestartRequest: type: object properties: config: $ref: '#/components/schemas/ApplicationConfig' UploadedFileInfo: type: object required: [location] properties: location: type: string ApplicationConfigValidation: type: object properties: clusters: type: object additionalProperties: $ref: '#/components/schemas/ClusterConfigValidation' ApplicationPropertyValidation: type: object required: [error] properties: error: type: boolean errorMessage: type: string description: Contains error message if error = true ClusterConfigValidation: type: object required: [kafka] properties: kafka: $ref: '#/components/schemas/ApplicationPropertyValidation' schemaRegistry: $ref: '#/components/schemas/ApplicationPropertyValidation' kafkaConnects: type: object additionalProperties: $ref: '#/components/schemas/ApplicationPropertyValidation' ksqldb: $ref: '#/components/schemas/ApplicationPropertyValidation' ApplicationConfig: type: object properties: properties: type: object properties: auth: type: object properties: type: type: string oauth2: type: object properties: client: type: object additionalProperties: type: object properties: provider: type: string clientId: type: string clientSecret: type: string clientName: type: string redirectUri: type: string authorizationGrantType: type: string issuerUri: type: string authorizationUri: type: string tokenUri: type: string userInfoUri: type: string jwkSetUri: type: string userNameAttribute: type: string scope: type: array items: type: string customParams: type: object additionalProperties: type: string rbac: type: object properties: roles: type: array items: type: object properties: name: type: string clusters: type: array items: type: string subjects: type: array items: type: object properties: provider: type: string type: type: string value: type: string permissions: type: array items: type: object properties: resource: $ref: '#/components/schemas/ResourceType' value: type: string actions: type: array items: $ref: '#/components/schemas/Action' webclient: type: object properties: maxInMemoryBufferSize: type: string description: "examples: 20, 12KB, 5MB" kafka: type: object properties: polling: type: object properties: pollTimeoutMs: type: integer maxPageSize: type: integer defaultPageSize: type: integer adminClientTimeout: type: integer internalTopicPrefix: type: string clusters: type: array items: type: object properties: name: type: string bootstrapServers: type: string ssl: type: object properties: truststoreLocation: type: string truststorePassword: type: string schemaRegistry: type: string schemaRegistryAuth: type: object properties: username: type: string password: type: string schemaRegistrySsl: type: object properties: keystoreLocation: type: string keystorePassword: type: string ksqldbServer: type: string ksqldbServerSsl: type: object properties: keystoreLocation: type: string keystorePassword: type: string ksqldbServerAuth: type: object properties: username: type: string password: type: string kafkaConnect: type: array items: type: object properties: name: type: string address: type: string username: type: string password: type: string keystoreLocation: type: string keystorePassword: type: string metrics: type: object properties: type: type: string port: type: integer format: int32 ssl: type: boolean username: type: string password: type: string keystoreLocation: type: string keystorePassword: type: string properties: type: object additionalProperties: true readOnly: type: boolean disableLogDirsCollection: type: boolean serde: type: array items: type: object properties: name: type: string className: type: string filePath: type: string properties: type: object additionalProperties: true topicKeysPattern: type: string topicValuesPattern: type: string defaultKeySerde: type: string defaultValueSerde: type: string masking: type: array items: type: object properties: type: type: string enum: - REMOVE - MASK - REPLACE fields: type: array items: type: string fieldsNamePattern: type: string maskingCharsReplacement: type: array items: type: string replacement: type: string topicKeysPattern: type: string topicValuesPattern: type: string pollingThrottleRate: type: integer format: int64 audit: type: object properties: level: type: string enum: [ "ALL", "ALTER_ONLY" ] topic: type: string auditTopicsPartitions: type: integer topicAuditEnabled: type: boolean consoleAuditEnabled: type: boolean auditTopicProperties: type: object additionalProperties: type: string ================================================ FILE: kafka-ui-e2e-checks/.gitignore ================================================ .env build/ allure-results/ selenoid/video/ target/ selenoid/logs/ ================================================ FILE: kafka-ui-e2e-checks/QASE.md ================================================ ### E2E integration with Qase.io TMS (for internal users) ### Table of Contents - [Intro](#intro) - [Set up Qase.io integration](#set-up-qase-integration) - [Test case creation](#test-case-creation) - [Test run reporting](#test-run-reporting) ### Intro We're using [Qase.io](https://help.qase.io/en/) as TMS to keep test cases and accumulate test runs. Integration is set up through API using [qase-api](https://mvnrepository.com/artifact/io.qase/qase-api) and [qase-testng](https://mvnrepository.com/artifact/io.qase/qase-testng) libraries. ### Set up Qase integration To set up integration locally add next VM option `-DQASEIO_API_TOKEN='%s'` (add your [Qase token](https://app.qase.io/user/api/token) instead of '%s') into your run configuration ### Test case creation All new test cases can be added into TMS by default if they have no QaseId and QaseTitle matching already existing cases. But to handle `@Suite` and `@Automation` we added custom QaseCreateListener. To create new test case for next sync with Qase (see example `kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qaseSuite/Template.java`): 1. Create new class in `kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qaseSuite/suit` 2. Inherit it from `kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qaseSuite/BaseQaseTest.java` 3. Create new test method with some name inside the class and annotate it with: - `@Automation` (optional - Not automated by default) - to set one of automation states: NOT_AUTOMATED, TO_BE_AUTOMATED, AUTOMATED - `@QaseTitle` (required) - to set title for new test case and to check is there no existing cases with same title in Qase.io - `@Status` (optional - Draft by default) - to set one of case statuses: ACTUAL, DRAFT, DEPRECATED - `@Suite` (optional) - to store new case in some existing package need to set its id, otherwise case will be stored in the root - `@Test` (required) - annotation from TestNG to specify this method as test 4. Create new private void step methods with some name inside the same class and annotate it with @io.qase.api.annotation.Step to specify this method as step. 5. Use defined step methods inside created test method in concrete order 6. If there are any additional cases to create you can repeat scenario in a new class 7. There are two ways to sync newly created cases in the framework with Qase.io: - sync can be performed locally - run new test classes with already [set up Qase.io integration](#Set up Qase.io integration) - also you can commit and push your changes, then run [E2E Manual suite](https://github.com/provectus/kafka-ui/actions/workflows/e2e-manual.yml) on your branch 8. No test run in Qase.io will be created, new test case will be stored defined directory in [project's repository](https://app.qase.io/project/KAFKAUI) 9. To add expected results into created test case edit in Qase.io manually ### Test run reporting To handle manual test cases with status `Skipped` we added custom QaseResultListener. To create new test run: 1. All test methods should be annotated with actual `@QaseId` 2. There are two ways to sync newly created cases in the framework with Qase.io: - run can be performed locally - run test classes (or suites) with already [set up Qase.io integration](#Set up Qase.io integration), they will be labeled as `Automation CUSTOM suite` - also you can commit and push your changes, then run [E2E Automation suite](https://github.com/provectus/kafka-ui/actions/workflows/e2e-automation.yml) on your branch 3. All new test runs will be added into [project's test runs](https://app.qase.io/run/KAFKAUI) with corresponding label using QaseId to identify existing cases 4. All test cases from manual suite are set up to have `Skipped` status in test runs to perform them manually ================================================ FILE: kafka-ui-e2e-checks/README.md ================================================ ### E2E UI automation for Kafka-ui This repository is for E2E UI automation. ### Table of Contents - [Prerequisites](#prerequisites) - [How to install](#how-to-install) - [How to run checks](#how-to-run-checks) - [Qase.io integration (for internal users)](#qase-integration) - [Reporting](#reporting) - [Environments setup](#environments-setup) - [Test Data](#test-data) - [Actions](#actions) - [Checks](#checks) - [Parallelization](#parallelization) - [How to develop](#how-to-develop) ### Prerequisites - Docker & Docker-compose - Java (install aarch64 jdk if you have M1/arm chip) - Maven ### How to install ``` git clone https://github.com/provectus/kafka-ui.git cd kafka-ui-e2e-checks docker pull selenoid/vnc_chrome:103.0 ``` ### How to run checks 1. Run `kafka-ui`: ``` cd kafka-ui docker-compose -f kafka-ui-e2e-checks/docker/selenoid-local.yaml up -d docker-compose -f documentation/compose/e2e-tests.yaml up -d ``` 2. To run test suite select its name (options: regression, sanity, smoke) and put it instead %s into command below ``` ./mvnw -Dsurefire.suiteXmlFiles='src/test/resources/%s.xml' -f 'kafka-ui-e2e-checks' test -Pprod ``` 3. To run tests on your local Chrome browser just add next VM option to the Run Configuration ``` -Dbrowser=local ``` Expected Location of Chrome ``` Linux: /usr/bin/google-chrome1 Mac: /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome Windows XP: %HOMEPATH%\Local Settings\Application Data\Google\Chrome\Application\chrome.exe Windows Vista and newer: C:\Users%USERNAME%\AppData\Local\Google\Chrome\Application\chrome.exe ``` ### Qase integration Found instruction for Qase.io integration (for internal use only) at `kafka-ui-e2e-checks/QASE.md` ### Reporting Reports are in `allure-results` folder. If you have installed allure commandline [here](https://www.npmjs.com/package/allure-commandline)) You can see allure report with command: ``` allure serve ``` ### Screenshots Reference screenshots are in `SCREENSHOTS_FOLDER` (default,`kafka-ui-e2e-checks/screenshots`) ### How to develop > ⚠️ todo ### Setting for different environments > ⚠️ todo ### Test Data > ⚠️ todo ### Actions > ⚠️ todo ### Checks > ⚠️ todo ### Parallelization > ⚠️ todo ### Tips - install `Selenium UI Testing plugin` in IDEA ================================================ FILE: kafka-ui-e2e-checks/docker/selenoid-git.yaml ================================================ --- version: '3' services: selenoid: network_mode: bridge image: aerokube/selenoid:1.10.7 volumes: - "../selenoid/config:/etc/selenoid" - "/var/run/docker.sock:/var/run/docker.sock" - "../selenoid/video:/opt/selenoid/video" - "../selenoid/logs:/opt/selenoid/logs" environment: - OVERRIDE_VIDEO_OUTPUT_DIR=../selenoid/video command: [ "-conf", "/etc/selenoid/browsersGit.json", "-video-output-dir", "/opt/selenoid/video", "-log-output-dir", "/opt/selenoid/logs" ] ports: - "4444:4444" selenoid-ui: network_mode: bridge image: aerokube/selenoid-ui:latest-release links: - selenoid ports: - "8081:8080" command: [ "--selenoid-uri", "http://selenoid:4444" ] selenoid-chrome: network_mode: bridge image: selenoid/vnc_chrome:103.0 extra_hosts: - "host.docker.internal:host-gateway" ================================================ FILE: kafka-ui-e2e-checks/docker/selenoid-local.yaml ================================================ --- version: '3' services: selenoid: network_mode: bridge image: aerokube/selenoid:1.10.7 volumes: - "../selenoid/config:/etc/selenoid" - "/var/run/docker.sock:/var/run/docker.sock" - "../selenoid/video:/opt/selenoid/video" - "../selenoid/logs:/opt/selenoid/logs" environment: - OVERRIDE_VIDEO_OUTPUT_DIR=../selenoid/video command: [ "-conf", "/etc/selenoid/browsersLocal.json", "-video-output-dir", "/opt/selenoid/video", "-log-output-dir", "/opt/selenoid/logs" ] ports: - "4444:4444" selenoid-ui: network_mode: bridge image: aerokube/selenoid-ui:latest-release links: - selenoid ports: - "8081:8080" command: [ "--selenoid-uri", "http://selenoid:4444" ] selenoid-chrome: network_mode: bridge image: selenoid/vnc_chrome:103.0 extra_hosts: - "host.docker.internal:host-gateway" ================================================ FILE: kafka-ui-e2e-checks/pom.xml ================================================ kafka-ui com.provectus 0.0.1-SNAPSHOT 4.0.0 kafka-ui-e2e-checks 3.0.0-M8 ${project.version} 1.17.6 5.2.1 4.8.1 6.12.3 7.7.1 2.23.0 3.0.5 1.9.9.1 3.24.2 2.2 2.0.7 3.3.1 org.apache.kafka kafka_2.13 ${kafka.version} io.netty netty-buffer io.netty netty-common io.netty netty-codec io.netty netty-handler io.netty netty-resolver io.netty netty-transport io.netty netty-transport-native-epoll io.netty netty-transport-native-unix-common io.netty netty-buffer io.netty netty-common io.netty netty-codec io.netty netty-handler io.netty netty-resolver io.netty netty-transport io.netty netty-transport-native-epoll io.netty netty-transport-native-unix-common io.netty netty-resolver-dns-native-macos osx-aarch_64 org.testcontainers testcontainers ${testcontainers.version} org.testcontainers selenium ${testcontainers.version} org.projectlombok lombok ${org.projectlombok.version} org.apache.httpcomponents.core5 httpcore5 ${httpcomponents.version} org.apache.httpcomponents.client5 httpclient5 ${httpcomponents.version} org.seleniumhq.selenium selenium-http-jdk-client ${selenium.version} org.seleniumhq.selenium selenium-http ${selenium.version} com.codeborne selenide ${selenide.version} org.testng testng ${testng.version} io.qameta.allure allure-selenide ${allure.version} io.qameta.allure allure-testng ${allure.version} io.qase qase-testng ${qase.io.version} io.qase qase-api ${qase.io.version} org.hamcrest hamcrest ${hamcrest.version} org.assertj assertj-core ${assertj.version} org.aspectj aspectjrt ${aspectj.version} org.slf4j slf4j-simple ${slf4j.version} com.provectus kafka-ui-contract ${kafka-ui-contract} local true org.apache.maven.plugins maven-surefire-plugin true org.apache.maven.surefire surefire-testng ${maven.surefire-plugin.version} org.apache.maven.plugins maven-compiler-plugin prod org.apache.maven.plugins maven-surefire-plugin ${maven.surefire-plugin.version} -javaagent:"${settings.localRepository}/org/aspectj/aspectjweaver/${aspectj.version}/aspectjweaver-${aspectj.version}.jar" org.apache.maven.surefire surefire-testng ${maven.surefire-plugin.version} org.aspectj aspectjweaver ${aspectj.version} io.qameta.allure allure-maven 2.10.0 org.apache.maven.plugins maven-checkstyle-plugin 3.3.0 com.puppycrawl.tools checkstyle 10.3.1 checkstyle validate check warning true true true file:${basedir}/../etc/checkstyle/checkstyle-e2e.xml file:${basedir}/../etc/checkstyle/apache-header.txt ================================================ FILE: kafka-ui-e2e-checks/selenoid/config/browsersGit.json ================================================ { "chrome": { "default": "103.0", "versions": { "103.0": { "image": "selenoid/vnc_chrome:103.0", "hosts": [ "host.docker.internal:172.17.0.1" ], "port": "4444", "path": "/" } } } } ================================================ FILE: kafka-ui-e2e-checks/selenoid/config/browsersLocal.json ================================================ { "chrome": { "default": "103.0", "versions": { "103.0": { "image": "selenoid/vnc_chrome:103.0", "port": "4444", "path": "/" } } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Connector.java ================================================ package com.provectus.kafka.ui.models; import lombok.Data; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Connector { private String name, config; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Schema.java ================================================ package com.provectus.kafka.ui.models; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.provectus.kafka.ui.api.model.SchemaType; import lombok.Data; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Schema { private static final String USER_DIR = "user.dir"; private String name, valuePath; private SchemaType type; public static Schema createSchemaAvro() { return new Schema().setName("schema_avro-" + randomAlphabetic(5)) .setType(SchemaType.AVRO) .setValuePath(System.getProperty(USER_DIR) + "/src/main/resources/testData/schemas/schema_avro_value.json"); } public static Schema createSchemaJson() { return new Schema().setName("schema_json-" + randomAlphabetic(5)) .setType(SchemaType.JSON) .setValuePath(System.getProperty(USER_DIR) + "/src/main/resources/testData/schemas/schema_json_Value.json"); } public static Schema createSchemaProtobuf() { return new Schema().setName("schema_protobuf-" + randomAlphabetic(5)) .setType(SchemaType.PROTOBUF) .setValuePath( System.getProperty(USER_DIR) + "/src/main/resources/testData/schemas/schema_protobuf_value.txt"); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/models/Topic.java ================================================ package com.provectus.kafka.ui.models; import com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue; import com.provectus.kafka.ui.pages.topics.enums.CustomParameterType; import com.provectus.kafka.ui.pages.topics.enums.MaxSizeOnDisk; import com.provectus.kafka.ui.pages.topics.enums.TimeToRetain; import lombok.Data; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Topic { private String name, timeToRetainData, maxMessageBytes, messageKey, messageValue, customParameterValue; private int numberOfPartitions; private CustomParameterType customParameterType; private CleanupPolicyValue cleanupPolicyValue; private MaxSizeOnDisk maxSizeOnDisk; private TimeToRetain timeToRetain; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/BasePage.java ================================================ package com.provectus.kafka.ui.pages; import static com.codeborne.selenide.Selenide.$$x; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.codeborne.selenide.ElementsCollection; import com.codeborne.selenide.SelenideElement; import com.codeborne.selenide.WebDriverRunner; import com.provectus.kafka.ui.pages.panels.enums.MenuItem; import com.provectus.kafka.ui.utilities.WebUtils; import java.time.Duration; import lombok.extern.slf4j.Slf4j; import org.openqa.selenium.Keys; import org.openqa.selenium.interactions.Actions; @Slf4j public abstract class BasePage extends WebUtils { protected SelenideElement loadingSpinner = $x("//div[@role='progressbar']"); protected SelenideElement submitBtn = $x("//button[@type='submit']"); protected SelenideElement tableGrid = $x("//table"); protected SelenideElement searchFld = $x("//input[@type='text'][contains(@id, ':r')]"); protected SelenideElement dotMenuBtn = $x("//button[@aria-label='Dropdown Toggle']"); protected SelenideElement alertHeader = $x("//div[@role='alert']//div[@role='heading']"); protected SelenideElement alertMessage = $x("//div[@role='alert']//div[@role='contentinfo']"); protected SelenideElement confirmationMdl = $x("//div[text()= 'Confirm the action']/.."); protected SelenideElement confirmBtn = $x("//button[contains(text(),'Confirm')]"); protected SelenideElement cancelBtn = $x("//button[contains(text(),'Cancel')]"); protected SelenideElement backBtn = $x("//button[contains(text(),'Back')]"); protected SelenideElement previousBtn = $x("//button[contains(text(),'Previous')]"); protected SelenideElement nextBtn = $x("//button[contains(text(),'Next')]"); protected ElementsCollection ddlOptions = $$x("//li[@value]"); protected ElementsCollection gridItems = $$x("//tr[@class]"); protected String summaryCellLocator = "//div[contains(text(),'%s')]"; protected String tableElementNameLocator = "//tbody//a[contains(text(),'%s')]"; protected String columnHeaderLocator = "//table//tr/th//div[text()='%s']"; protected String pageTitleFromHeader = "//h1[text()='%s']"; protected String pagePathFromHeader = "//a[text()='%s']/../h1"; protected boolean isSpinnerVisible(int... timeoutInSeconds) { return isVisible(loadingSpinner, timeoutInSeconds); } protected void waitUntilSpinnerDisappear(int... timeoutInSeconds) { log.debug("\nwaitUntilSpinnerDisappear"); if (isSpinnerVisible(timeoutInSeconds)) { loadingSpinner.shouldBe(Condition.disappear, Duration.ofSeconds(60)); } } protected void searchItem(String tag) { log.debug("\nsearchItem: {}", tag); sendKeysAfterClear(searchFld, tag); searchFld.pressEnter().shouldHave(Condition.value(tag)); waitUntilSpinnerDisappear(1); } protected SelenideElement getPageTitleFromHeader(MenuItem menuItem) { return $x(String.format(pageTitleFromHeader, menuItem.getPageTitle())); } protected SelenideElement getPagePathFromHeader(MenuItem menuItem) { return $x(String.format(pagePathFromHeader, menuItem.getPageTitle())); } protected void clickSubmitBtn() { clickByJavaScript(submitBtn); } protected void clickNextBtn() { clickByJavaScript(nextBtn); } protected void clickBackBtn() { clickByJavaScript(backBtn); } protected void clickPreviousBtn() { clickByJavaScript(previousBtn); } protected void setJsonInputValue(SelenideElement jsonInput, String jsonConfig) { sendKeysByActions(jsonInput, jsonConfig.replace(" ", "")); new Actions(WebDriverRunner.getWebDriver()) .keyDown(Keys.SHIFT) .sendKeys(Keys.PAGE_DOWN) .keyUp(Keys.SHIFT) .sendKeys(Keys.DELETE) .perform(); } protected SelenideElement getTableElement(String elementName) { log.debug("\ngetTableElement: {}", elementName); return $x(String.format(tableElementNameLocator, elementName)); } protected ElementsCollection getDdlOptions() { return ddlOptions; } protected String getAlertHeader() { log.debug("\ngetAlertHeader"); String result = alertHeader.shouldBe(Condition.visible).getText(); log.debug("-> {}", result); return result; } protected String getAlertMessage() { log.debug("\ngetAlertMessage"); String result = alertMessage.shouldBe(Condition.visible).getText(); log.debug("-> {}", result); return result; } protected boolean isAlertVisible(AlertHeader header) { log.debug("\nisAlertVisible: {}", header.toString()); boolean result = getAlertHeader().equals(header.toString()); log.debug("-> {}", result); return result; } protected boolean isAlertVisible(AlertHeader header, String message) { log.debug("\nisAlertVisible: {} {}", header, message); boolean result = isAlertVisible(header) && getAlertMessage().equals(message); log.debug("-> {}", result); return result; } protected void clickConfirmButton() { confirmBtn.shouldBe(Condition.enabled).click(); confirmBtn.shouldBe(Condition.disappear); } protected void clickCancelButton() { cancelBtn.shouldBe(Condition.enabled).click(); cancelBtn.shouldBe(Condition.disappear); } protected boolean isConfirmationModalVisible() { return isVisible(confirmationMdl); } public enum AlertHeader { SUCCESS("Success"), VALIDATION_ERROR("Validation Error"), BAD_REQUEST("400 Bad Request"); private final String value; AlertHeader(String value) { this.value = value; } public String toString() { return value; } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersConfigTab.java ================================================ package com.provectus.kafka.ui.pages.brokers; import static com.codeborne.selenide.Selenide.$$x; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.ElementsCollection; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; public class BrokersConfigTab extends BasePage { protected List editBtn = $$x("//button[@aria-label='editAction']"); protected SelenideElement searchByKeyField = $x("//input[@placeholder='Search by Key or Value']"); protected SelenideElement sourceInfoIcon = $x("//div[text()='Source']/..//div/div[@class]"); protected SelenideElement sourceInfoTooltip = $x("//div[text()='Source']/..//div/div[@style]"); protected ElementsCollection editBtns = $$x("//button[@aria-label='editAction']"); @Step public BrokersConfigTab waitUntilScreenReady() { waitUntilSpinnerDisappear(); searchFld.shouldBe(Condition.visible); return this; } @Step public BrokersConfigTab hoverOnSourceInfoIcon() { sourceInfoIcon.shouldBe(Condition.visible).hover(); return this; } @Step public String getSourceInfoTooltipText() { return sourceInfoTooltip.shouldBe(Condition.visible).getText().trim(); } @Step public boolean isSearchByKeyVisible() { return isVisible(searchFld); } @Step public BrokersConfigTab searchConfig(String key) { searchItem(key); return this; } public List getColumnHeaders() { return Stream.of("Key", "Value", "Source") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } public List getEditButtons() { return editBtns; } @Step public BrokersConfigTab clickNextButton() { clickNextBtn(); waitUntilSpinnerDisappear(1); return this; } @Step public BrokersConfigTab clickPreviousButton() { clickPreviousBtn(); waitUntilSpinnerDisappear(1); return this; } private List initGridItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new BrokersConfigTab.BrokersConfigItem(item))); return gridItemList; } @Step public BrokersConfigTab.BrokersConfigItem getConfig(String key) { return initGridItems().stream() .filter(e -> e.getKey().equals(key)) .findFirst().orElseThrow(); } @Step public List getAllConfigs() { return initGridItems(); } public static class BrokersConfigItem extends BasePage { private final SelenideElement element; public BrokersConfigItem(SelenideElement element) { this.element = element; } @Step public String getKey() { return element.$x("./td[1]").getText().trim(); } @Step public String getValue() { return element.$x("./td[2]//span").getText().trim(); } @Step public BrokersConfigItem setValue(String value) { sendKeysAfterClear(getValueFld(), value); return this; } @Step public SelenideElement getValueFld() { return element.$x("./td[2]//input"); } @Step public SelenideElement getSaveBtn() { return element.$x("./td[2]//button[@aria-label='confirmAction']"); } @Step public SelenideElement getCancelBtn() { return element.$x("./td[2]//button[@aria-label='cancelAction']"); } @Step public SelenideElement getEditBtn() { return element.$x("./td[2]//button[@aria-label='editAction']"); } @Step public BrokersConfigItem clickSaveBtn() { getSaveBtn().shouldBe(Condition.enabled).click(); return this; } @Step public BrokersConfigItem clickCancelBtn() { getCancelBtn().shouldBe(Condition.enabled).click(); return this; } @Step public BrokersConfigItem clickEditBtn() { getEditBtn().shouldBe(Condition.enabled).click(); return this; } @Step public String getSource() { return element.$x("./td[3]").getText().trim(); } @Step public BrokersConfigItem clickConfirm() { clickConfirmButton(); return this; } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersDetails.java ================================================ package com.provectus.kafka.ui.pages.brokers; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; public class BrokersDetails extends BasePage { protected String brokersTabLocator = "//a[text()='%s']"; @Step public BrokersDetails waitUntilScreenReady() { waitUntilSpinnerDisappear(); $x(String.format(brokersTabLocator, DetailsTab.LOG_DIRECTORIES)).shouldBe(Condition.visible); return this; } @Step public BrokersDetails openDetailsTab(DetailsTab menu) { $x(String.format(brokersTabLocator, menu.toString())).shouldBe(Condition.enabled).click(); waitUntilSpinnerDisappear(); return this; } private List getVisibleColumnHeaders() { return Stream.of("Name", "Topics", "Error", "Partitions") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } private List getEnabledColumnHeaders() { return Stream.of("Name", "Error") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } private List getVisibleSummaryCells() { return Stream.of("Segment Size", "Segment Count", "Port", "Host") .map(name -> $x(String.format(summaryCellLocator, name))) .collect(Collectors.toList()); } private List getDetailsTabs() { return Stream.of(DetailsTab.values()) .map(name -> $x(String.format(brokersTabLocator, name))) .collect(Collectors.toList()); } @Step public List getAllEnabledElements() { List enabledElements = new ArrayList<>(getEnabledColumnHeaders()); enabledElements.addAll(getDetailsTabs()); return enabledElements; } @Step public List getAllVisibleElements() { List visibleElements = new ArrayList<>(getVisibleSummaryCells()); visibleElements.addAll(getVisibleColumnHeaders()); visibleElements.addAll(getDetailsTabs()); return visibleElements; } public enum DetailsTab { LOG_DIRECTORIES("Log directories"), CONFIGS("Configs"), METRICS("Metrics"); private final String value; DetailsTab(String value) { this.value = value; } public String toString() { return value; } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/brokers/BrokersList.java ================================================ package com.provectus.kafka.ui.pages.brokers; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.BROKERS; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; public class BrokersList extends BasePage { @Step public BrokersList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(BROKERS).shouldBe(Condition.visible); return this; } @Step public BrokersList openBroker(int brokerId) { getBroker(brokerId).openItem(); return this; } private List getUptimeSummaryCells() { return Stream.of("Broker Count", "Active Controller", "Version") .map(name -> $x(String.format(summaryCellLocator, name))) .collect(Collectors.toList()); } private List getPartitionsSummaryCells() { return Stream.of("Online", "URP", "In Sync Replicas", "Out Of Sync Replicas") .map(name -> $x(String.format(summaryCellLocator, name))) .collect(Collectors.toList()); } @Step public List getAllVisibleElements() { List visibleElements = new ArrayList<>(getUptimeSummaryCells()); visibleElements.addAll(getPartitionsSummaryCells()); return visibleElements; } private List getEnabledColumnHeaders() { return Stream.of("Broker ID", "Disk usage", "Partitions skew", "Leaders", "Leader skew", "Online partitions", "Port", "Host") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } @Step public List getAllEnabledElements() { return getEnabledColumnHeaders(); } private List initGridItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new BrokersGridItem(item))); return gridItemList; } @Step public BrokersGridItem getBroker(int id) { return initGridItems().stream() .filter(e -> e.getId() == id) .findFirst().orElseThrow(); } @Step public List getAllBrokers() { return initGridItems(); } public static class BrokersGridItem extends BasePage { private final SelenideElement element; public BrokersGridItem(SelenideElement element) { this.element = element; } private SelenideElement getIdElm() { return element.$x("./td[1]/div/a"); } @Step public int getId() { return Integer.parseInt(getIdElm().getText().trim()); } @Step public void openItem() { getIdElm().click(); } @Step public int getSegmentSize() { return Integer.parseInt(element.$x("./td[2]").getText().trim()); } @Step public int getSegmentCount() { return Integer.parseInt(element.$x("./td[3]").getText().trim()); } @Step public int getPort() { return Integer.parseInt(element.$x("./td[4]").getText().trim()); } @Step public String getHost() { return element.$x("./td[5]").getText().trim(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connectors/ConnectorCreateForm.java ================================================ package com.provectus.kafka.ui.pages.connectors; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class ConnectorCreateForm extends BasePage { protected SelenideElement nameField = $x("//input[@name='name']"); protected SelenideElement contentTextArea = $x("//textarea[@class='ace_text-input']"); protected SelenideElement configField = $x("//div[@id='config']"); @Step public ConnectorCreateForm waitUntilScreenReady() { waitUntilSpinnerDisappear(); nameField.shouldBe(Condition.visible); return this; } @Step public ConnectorCreateForm setName(String connectName) { nameField.shouldBe(Condition.enabled).setValue(connectName); return this; } @Step public ConnectorCreateForm setConfig(String configJson) { configField.shouldBe(Condition.enabled).click(); setJsonInputValue(contentTextArea, configJson); return this; } @Step public ConnectorCreateForm setConnectorDetails(String connectName, String configJson) { setName(connectName); setConfig(configJson); return this; } @Step public ConnectorCreateForm clickSubmitButton() { clickSubmitBtn(); waitUntilSpinnerDisappear(); return this; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connectors/ConnectorDetails.java ================================================ package com.provectus.kafka.ui.pages.connectors; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class ConnectorDetails extends BasePage { protected SelenideElement deleteBtn = $x("//li/div[contains(text(),'Delete')]"); protected SelenideElement confirmBtnMdl = $x("//div[@role='dialog']//button[contains(text(),'Confirm')]"); protected SelenideElement contentTextArea = $x("//textarea[@class='ace_text-input']"); protected SelenideElement taskTab = $x("//a[contains(text(),'Tasks')]"); protected SelenideElement configTab = $x("//a[contains(text(),'Config')]"); protected SelenideElement configField = $x("//div[@id='config']"); protected String connectorHeaderLocator = "//h1[contains(text(),'%s')]"; @Step public ConnectorDetails waitUntilScreenReady() { waitUntilSpinnerDisappear(); dotMenuBtn.shouldBe(Condition.visible); return this; } @Step public ConnectorDetails openConfigTab() { clickByJavaScript(configTab); return this; } @Step public ConnectorDetails setConfig(String configJson) { configField.shouldBe(Condition.enabled).click(); clearByKeyboard(contentTextArea); contentTextArea.setValue(configJson); configField.shouldBe(Condition.enabled).click(); return this; } @Step public ConnectorDetails clickSubmitButton() { clickSubmitBtn(); return this; } @Step public ConnectorDetails openDotMenu() { clickByJavaScript(dotMenuBtn); return this; } @Step public ConnectorDetails clickDeleteBtn() { clickByJavaScript(deleteBtn); return this; } @Step public ConnectorDetails clickConfirmBtn() { confirmBtnMdl.shouldBe(Condition.enabled).click(); confirmBtnMdl.shouldBe(Condition.disappear); return this; } @Step public ConnectorDetails deleteConnector() { openDotMenu(); clickDeleteBtn(); clickConfirmBtn(); return this; } @Step public boolean isConnectorHeaderVisible(String connectorName) { return isVisible($x(String.format(connectorHeaderLocator, connectorName))); } @Step public boolean isAlertWithMessageVisible(AlertHeader header, String message) { return isAlertVisible(header, message); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/connectors/KafkaConnectList.java ================================================ package com.provectus.kafka.ui.pages.connectors; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KAFKA_CONNECT; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class KafkaConnectList extends BasePage { protected SelenideElement createConnectorBtn = $x("//button[contains(text(),'Create Connector')]"); public KafkaConnectList() { tableElementNameLocator = "//tbody//td[contains(text(),'%s')]"; } @Step public KafkaConnectList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(KAFKA_CONNECT).shouldBe(Condition.visible); return this; } @Step public KafkaConnectList clickCreateConnectorBtn() { clickByJavaScript(createConnectorBtn); return this; } @Step public KafkaConnectList openConnector(String connectorName) { getTableElement(connectorName).shouldBe(Condition.enabled).click(); return this; } @Step public boolean isConnectorVisible(String connectorName) { tableGrid.shouldBe(Condition.visible); return isVisible(getTableElement(connectorName)); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/consumers/ConsumersDetails.java ================================================ package com.provectus.kafka.ui.pages.consumers; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class ConsumersDetails extends BasePage { protected String consumerIdHeaderLocator = "//h1[contains(text(),'%s')]"; protected String topicElementLocator = "//tbody//td//a[text()='%s']"; @Step public ConsumersDetails waitUntilScreenReady() { waitUntilSpinnerDisappear(); tableGrid.shouldBe(Condition.visible); return this; } @Step public boolean isRedirectedConsumerTitleVisible(String consumerGroupId) { return isVisible($x(String.format(consumerIdHeaderLocator, consumerGroupId))); } @Step public boolean isTopicInConsumersDetailsVisible(String topicName) { tableGrid.shouldBe(Condition.visible); return isVisible($x(String.format(topicElementLocator, topicName))); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/consumers/ConsumersList.java ================================================ package com.provectus.kafka.ui.pages.consumers; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.CONSUMERS; import com.codeborne.selenide.Condition; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class ConsumersList extends BasePage { @Step public ConsumersList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(CONSUMERS).shouldBe(Condition.visible); return this; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlDbList.java ================================================ package com.provectus.kafka.ui.pages.ksqldb; import static com.codeborne.selenide.Condition.visible; import static com.codeborne.selenide.Selenide.$; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KSQL_DB; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import com.provectus.kafka.ui.pages.ksqldb.enums.KsqlMenuTabs; import io.qameta.allure.Step; import java.time.Duration; import java.util.ArrayList; import java.util.List; import org.openqa.selenium.By; public class KsqlDbList extends BasePage { protected SelenideElement executeKsqlBtn = $x("//button[text()='Execute KSQL Request']"); protected SelenideElement tablesTab = $x("//nav[@role='navigation']/a[text()='Tables']"); protected SelenideElement streamsTab = $x("//nav[@role='navigation']/a[text()='Streams']"); @Step public KsqlDbList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(KSQL_DB).shouldBe(Condition.visible); return this; } @Step public KsqlDbList clickExecuteKsqlRequestBtn() { clickByJavaScript(executeKsqlBtn); return this; } @Step public KsqlDbList openDetailsTab(KsqlMenuTabs menu) { $(By.linkText(menu.toString())).shouldBe(Condition.visible).click(); waitUntilSpinnerDisappear(); return this; } private List initTablesItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new KsqlDbList.KsqlTablesGridItem(item))); return gridItemList; } @Step public KsqlDbList.KsqlTablesGridItem getTableByName(String tableName) { return initTablesItems().stream() .filter(e -> e.getTableName().equals(tableName)) .findFirst().orElseThrow(); } private List initStreamsItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new KsqlDbList.KsqlStreamsGridItem(item))); return gridItemList; } @Step public KsqlDbList.KsqlStreamsGridItem getStreamByName(String streamName) { return initStreamsItems().stream() .filter(e -> e.getStreamName().equals(streamName)) .findFirst().orElseThrow(); } public static class KsqlTablesGridItem extends BasePage { private final SelenideElement element; public KsqlTablesGridItem(SelenideElement element) { this.element = element; } private SelenideElement getNameElm() { return element.$x("./td[1]"); } @Step public String getTableName() { return getNameElm().getText().trim(); } @Step public boolean isVisible() { boolean isVisible = false; try { getNameElm().shouldBe(visible, Duration.ofMillis(500)); isVisible = true; } catch (Throwable ignored) { } return isVisible; } @Step public String getTopicName() { return element.$x("./td[2]").getText().trim(); } @Step public String getKeyFormat() { return element.$x("./td[3]").getText().trim(); } @Step public String getValueFormat() { return element.$x("./td[4]").getText().trim(); } @Step public String getIsWindowed() { return element.$x("./td[5]").getText().trim(); } } public static class KsqlStreamsGridItem extends BasePage { private final SelenideElement element; public KsqlStreamsGridItem(SelenideElement element) { this.element = element; } private SelenideElement getNameElm() { return element.$x("./td[1]"); } @Step public String getStreamName() { return getNameElm().getText().trim(); } @Step public boolean isVisible() { boolean isVisible = false; try { getNameElm().shouldBe(visible, Duration.ofMillis(500)); isVisible = true; } catch (Throwable ignored) { } return isVisible; } @Step public String getTopicName() { return element.$x("./td[2]").getText().trim(); } @Step public String getKeyFormat() { return element.$x("./td[3]").getText().trim(); } @Step public String getValueFormat() { return element.$x("./td[4]").getText().trim(); } @Step public String getIsWindowed() { return element.$x("./td[5]").getText().trim(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/KsqlQueryForm.java ================================================ package com.provectus.kafka.ui.pages.ksqldb; import static com.codeborne.selenide.Condition.visible; import static com.codeborne.selenide.Selenide.$$x; import static com.codeborne.selenide.Selenide.$x; import static com.codeborne.selenide.Selenide.sleep; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.ElementsCollection; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.time.Duration; import java.util.ArrayList; import java.util.List; public class KsqlQueryForm extends BasePage { protected SelenideElement clearBtn = $x("//div/button[text()='Clear']"); protected SelenideElement executeBtn = $x("//div/button[text()='Execute']"); protected SelenideElement clearResultsBtn = $x("//div/button[text()='Clear results']"); protected SelenideElement addStreamPropertyBtn = $x("//button[text()='Add Stream Property']"); protected SelenideElement queryAreaValue = $x("//div[@class='ace_content']"); protected SelenideElement queryArea = $x("//div[@id='ksql']/textarea[@class='ace_text-input']"); protected SelenideElement abortButton = $x("//div[@role='status']/div[text()='Abort']"); protected SelenideElement cancelledAlert = $x("//div[@role='status'][text()='Cancelled']"); protected ElementsCollection ksqlGridItems = $$x("//tbody//tr"); protected ElementsCollection keyField = $$x("//input[@aria-label='key']"); protected ElementsCollection valueField = $$x("//input[@aria-label='value']"); @Step public KsqlQueryForm waitUntilScreenReady() { waitUntilSpinnerDisappear(); executeBtn.shouldBe(Condition.visible); return this; } @Step public KsqlQueryForm clickClearBtn() { clickByJavaScript(clearBtn); sleep(500); return this; } @Step public String getEnteredQuery() { return queryAreaValue.getText().trim(); } @Step public KsqlQueryForm clickExecuteBtn(String query) { clickByActions(executeBtn); if (query.contains("EMIT CHANGES")) { abortButton.shouldBe(Condition.visible); } else { waitUntilSpinnerDisappear(); } return this; } @Step public boolean isAbortBtnVisible() { return isVisible(abortButton); } @Step public KsqlQueryForm clickAbortBtn() { clickByActions(abortButton); return this; } @Step public boolean isCancelledAlertVisible() { return isVisible(cancelledAlert); } @Step public boolean isClearResultsBtnEnabled() { return isEnabled(clearResultsBtn); } @Step public KsqlQueryForm clickClearResultsBtn() { clickByActions(clearResultsBtn); waitUntilSpinnerDisappear(); return this; } @Step public KsqlQueryForm clickAddStreamProperty() { clickByActions(addStreamPropertyBtn); return this; } @Step public KsqlQueryForm setQuery(String query) { queryAreaValue.shouldBe(Condition.visible).click(); sendKeysByActions(queryArea, query); return this; } @Step public KsqlQueryForm.KsqlResponseGridItem getItemByName(String name) { return initItems().stream() .filter(e -> e.getName().equalsIgnoreCase(name)) .findFirst().orElseThrow(); } @Step public boolean areResultsVisible() { boolean visible = false; try { visible = initItems().size() > 0; } catch (Throwable ignored) { } return visible; } private List initItems() { List gridItemList = new ArrayList<>(); ksqlGridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new KsqlQueryForm.KsqlResponseGridItem(item))); return gridItemList; } public static class KsqlResponseGridItem extends BasePage { private final SelenideElement element; private KsqlResponseGridItem(SelenideElement element) { this.element = element; } @Step public String getType() { return element.$x("./td[1]").getText().trim(); } private SelenideElement getNameElm() { return element.$x("./td[2]"); } @Step public String getName() { return getNameElm().scrollTo().getText().trim(); } @Step public boolean isVisible() { boolean isVisible = false; try { getNameElm().shouldBe(visible, Duration.ofMillis(500)); isVisible = true; } catch (Throwable ignored) { } return isVisible; } @Step public String getTopic() { return element.$x("./td[3]").getText().trim(); } @Step public String getKeyFormat() { return element.$x("./td[4]").getText().trim(); } @Step public String getValueFormat() { return element.$x("./td[5]").getText().trim(); } @Step public String getIsWindowed() { return element.$x("./td[6]").getText().trim(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/enums/KsqlMenuTabs.java ================================================ package com.provectus.kafka.ui.pages.ksqldb.enums; public enum KsqlMenuTabs { TABLES("Table"), STREAMS("Streams"); private final String value; KsqlMenuTabs(String value) { this.value = value; } public String toString() { return value; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/enums/KsqlQueryConfig.java ================================================ package com.provectus.kafka.ui.pages.ksqldb.enums; public enum KsqlQueryConfig { SHOW_TABLES("show tables;"), SHOW_STREAMS("show streams;"), SELECT_ALL_FROM("SELECT * FROM %s\n" + "EMIT CHANGES;"); private final String query; KsqlQueryConfig(String query) { this.query = query; } public String getQuery() { return query; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/models/Stream.java ================================================ package com.provectus.kafka.ui.pages.ksqldb.models; import lombok.Data; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Stream { private String name, topicName, valueFormat, partitions; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/ksqldb/models/Table.java ================================================ package com.provectus.kafka.ui.pages.ksqldb.models; import lombok.Data; import lombok.experimental.Accessors; @Data @Accessors(chain = true) public class Table { private String name, streamName; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/panels/NaviSideBar.java ================================================ package com.provectus.kafka.ui.pages.panels; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.settings.BaseSource.CLUSTER_NAME; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import com.provectus.kafka.ui.pages.panels.enums.MenuItem; import io.qameta.allure.Step; import java.time.Duration; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; public class NaviSideBar extends BasePage { protected SelenideElement dashboardMenuItem = $x("//a[@title='Dashboard']"); protected String sideMenuOptionElementLocator = ".//ul/li[contains(.,'%s')]"; protected String clusterElementLocator = "//aside/ul/li[contains(.,'%s')]"; private SelenideElement expandCluster(String clusterName) { SelenideElement clusterElement = $x(String.format(clusterElementLocator, clusterName)).shouldBe(Condition.visible); if (clusterElement.parent().$$x(".//ul").size() == 0) { clickByActions(clusterElement); } return clusterElement; } @Step public NaviSideBar waitUntilScreenReady() { waitUntilSpinnerDisappear(); dashboardMenuItem.shouldBe(Condition.visible, Duration.ofSeconds(30)); return this; } @Step public String getPagePath(MenuItem menuItem) { return getPagePathFromHeader(menuItem) .shouldBe(Condition.visible) .getText().trim(); } @Step public NaviSideBar openSideMenu(String clusterName, MenuItem menuItem) { clickByActions(expandCluster(clusterName).parent() .$x(String.format(sideMenuOptionElementLocator, menuItem.getNaviTitle()))); return this; } @Step public NaviSideBar openSideMenu(MenuItem menuItem) { openSideMenu(CLUSTER_NAME, menuItem); return this; } public List getAllMenuButtons() { expandCluster(CLUSTER_NAME); return Stream.of(MenuItem.values()) .map(menuItem -> $x(String.format(sideMenuOptionElementLocator, menuItem.getNaviTitle()))) .collect(Collectors.toList()); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/panels/TopPanel.java ================================================ package com.provectus.kafka.ui.pages.panels; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import java.util.Arrays; import java.util.List; public class TopPanel extends BasePage { protected SelenideElement kafkaLogo = $x("//a[contains(text(),'UI for Apache Kafka')]"); protected SelenideElement kafkaVersion = $x("//a[@title='Current commit']"); protected SelenideElement logOutBtn = $x("//button[contains(text(),'Log out')]"); protected SelenideElement gitBtn = $x("//a[@href='https://github.com/provectus/kafka-ui']"); protected SelenideElement discordBtn = $x("//a[contains(@href,'https://discord.com/invite')]"); public List getAllVisibleElements() { return Arrays.asList(kafkaLogo, kafkaVersion, gitBtn, discordBtn); } public List getAllEnabledElements() { return Arrays.asList(gitBtn, discordBtn, kafkaLogo); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/panels/enums/MenuItem.java ================================================ package com.provectus.kafka.ui.pages.panels.enums; public enum MenuItem { DASHBOARD("Dashboard", "Dashboard"), BROKERS("Brokers", "Brokers"), TOPICS("Topics", "Topics"), CONSUMERS("Consumers", "Consumers"), SCHEMA_REGISTRY("Schema Registry", "Schema Registry"), KAFKA_CONNECT("Kafka Connect", "Connectors"), KSQL_DB("KSQL DB", "KSQL DB"); private final String naviTitle; private final String pageTitle; MenuItem(String naviTitle, String pageTitle) { this.naviTitle = naviTitle; this.pageTitle = pageTitle; } public String getNaviTitle() { return naviTitle; } public String getPageTitle() { return pageTitle; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schemas/SchemaCreateForm.java ================================================ package com.provectus.kafka.ui.pages.schemas; import static com.codeborne.selenide.Selenide.$; import static com.codeborne.selenide.Selenide.$$x; import static com.codeborne.selenide.Selenide.$x; import static org.openqa.selenium.By.id; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.codeborne.selenide.WebDriverRunner; import com.provectus.kafka.ui.api.model.CompatibilityLevel; import com.provectus.kafka.ui.api.model.SchemaType; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openqa.selenium.Keys; import org.openqa.selenium.interactions.Actions; public class SchemaCreateForm extends BasePage { protected SelenideElement schemaNameField = $x("//input[@name='subject']"); protected SelenideElement pageTitle = $x("//h1['Edit']"); protected SelenideElement schemaTextArea = $x("//textarea[@name='schema']"); protected SelenideElement newSchemaInput = $("#newSchema [wrap]"); protected SelenideElement schemaTypeDdl = $x("//ul[@name='schemaType']"); protected SelenideElement compatibilityLevelList = $x("//ul[@name='compatibilityLevel']"); protected SelenideElement newSchemaTextArea = $x("//div[@id='newSchema']"); protected SelenideElement latestSchemaTextArea = $x("//div[@id='latestSchema']"); protected SelenideElement leftVersionDdl = $(id("left-select")); protected SelenideElement rightVersionDdl = $(id("right-select")); protected List visibleMarkers = $$x("//div[@class='ace_scroller']//div[contains(@class,'codeMarker')]"); protected List elementsCompareVersionDdl = $$x("//ul[@role='listbox']/ul/li"); protected String ddlElementLocator = "//li[@value='%s']"; @Step public SchemaCreateForm waitUntilScreenReady() { waitUntilSpinnerDisappear(); pageTitle.shouldBe(Condition.visible); return this; } @Step public SchemaCreateForm setSubjectName(String name) { schemaNameField.setValue(name); return this; } @Step public SchemaCreateForm setSchemaField(String text) { schemaTextArea.setValue(text); return this; } @Step public SchemaCreateForm selectSchemaTypeFromDropdown(SchemaType schemaType) { schemaTypeDdl.shouldBe(Condition.enabled).click(); $x(String.format(ddlElementLocator, schemaType.getValue())).shouldBe(Condition.visible).click(); return this; } @Step public SchemaCreateForm clickSubmitButton() { clickSubmitBtn(); return this; } @Step public SchemaCreateForm selectCompatibilityLevelFromDropdown(CompatibilityLevel.CompatibilityEnum level) { compatibilityLevelList.shouldBe(Condition.enabled).click(); $x(String.format(ddlElementLocator, level.getValue())).shouldBe(Condition.visible).click(); return this; } @Step public SchemaCreateForm openLeftVersionDdl() { leftVersionDdl.shouldBe(Condition.enabled).click(); return this; } @Step public SchemaCreateForm openRightVersionDdl() { rightVersionDdl.shouldBe(Condition.enabled).click(); return this; } @Step public int getVersionsNumberFromList() { return elementsCompareVersionDdl.size(); } @Step public SchemaCreateForm selectVersionFromDropDown(int versionNumberDd) { $x(String.format(ddlElementLocator, versionNumberDd)).shouldBe(Condition.visible).click(); return this; } @Step public int getMarkedLinesNumber() { return visibleMarkers.size(); } @Step public SchemaCreateForm setNewSchemaValue(String configJson) { newSchemaTextArea.shouldBe(Condition.visible).click(); newSchemaInput.shouldBe(Condition.enabled); new Actions(WebDriverRunner.getWebDriver()) .sendKeys(Keys.PAGE_UP) .keyDown(Keys.SHIFT) .sendKeys(Keys.PAGE_DOWN) .keyUp(Keys.SHIFT) .sendKeys(Keys.DELETE) .perform(); setJsonInputValue(newSchemaInput, configJson); return this; } @Step public List getAllDetailsPageElements() { return Stream.of(compatibilityLevelList, newSchemaTextArea, latestSchemaTextArea, submitBtn, schemaTypeDdl) .collect(Collectors.toList()); } @Step public boolean isSubmitBtnEnabled() { return isEnabled(submitBtn); } @Step public boolean isSchemaDropDownEnabled() { boolean enabled = true; try { String attribute = schemaTypeDdl.getAttribute("disabled"); enabled = false; } catch (Throwable ignored) { } return enabled; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schemas/SchemaDetails.java ================================================ package com.provectus.kafka.ui.pages.schemas; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class SchemaDetails extends BasePage { protected SelenideElement actualVersionTextArea = $x("//div[@id='schema']"); protected SelenideElement compatibilityField = $x("//h4[contains(text(),'Compatibility')]/../p"); protected SelenideElement editSchemaBtn = $x("//button[contains(text(),'Edit Schema')]"); protected SelenideElement removeBtn = $x("//*[contains(text(),'Remove')]"); protected SelenideElement schemaConfirmBtn = $x("//div[@role='dialog']//button[contains(text(),'Confirm')]"); protected SelenideElement schemaTypeField = $x("//h4[contains(text(),'Type')]/../p"); protected SelenideElement latestVersionField = $x("//h4[contains(text(),'Latest version')]/../p"); protected SelenideElement compareVersionBtn = $x("//button[text()='Compare Versions']"); protected String schemaHeaderLocator = "//h1[contains(text(),'%s')]"; @Step public SchemaDetails waitUntilScreenReady() { waitUntilSpinnerDisappear(); actualVersionTextArea.shouldBe(Condition.visible); return this; } @Step public String getCompatibility() { return compatibilityField.getText(); } @Step public boolean isSchemaHeaderVisible(String schemaName) { return isVisible($x(String.format(schemaHeaderLocator, schemaName))); } @Step public int getLatestVersion() { return Integer.parseInt(latestVersionField.getText()); } @Step public String getSchemaType() { return schemaTypeField.getText(); } @Step public SchemaDetails openEditSchema() { editSchemaBtn.shouldBe(Condition.visible).click(); return this; } @Step public SchemaDetails openCompareVersionMenu() { compareVersionBtn.shouldBe(Condition.enabled).click(); return this; } @Step public SchemaDetails removeSchema() { clickByJavaScript(dotMenuBtn); removeBtn.shouldBe(Condition.enabled).click(); schemaConfirmBtn.shouldBe(Condition.visible).click(); schemaConfirmBtn.shouldBe(Condition.disappear); return this; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/schemas/SchemaRegistryList.java ================================================ package com.provectus.kafka.ui.pages.schemas; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.SCHEMA_REGISTRY; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; public class SchemaRegistryList extends BasePage { protected SelenideElement createSchemaBtn = $x("//button[contains(text(),'Create Schema')]"); @Step public SchemaRegistryList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(SCHEMA_REGISTRY).shouldBe(Condition.visible); return this; } @Step public SchemaRegistryList clickCreateSchema() { clickByJavaScript(createSchemaBtn); return this; } @Step public SchemaRegistryList openSchema(String schemaName) { getTableElement(schemaName) .shouldBe(Condition.enabled).click(); return this; } @Step public boolean isSchemaVisible(String schemaName) { tableGrid.shouldBe(Condition.visible); return isVisible(getTableElement(schemaName)); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/ProduceMessagePanel.java ================================================ package com.provectus.kafka.ui.pages.topics; import static com.codeborne.selenide.Selenide.$x; import static com.codeborne.selenide.Selenide.refresh; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.Arrays; public class ProduceMessagePanel extends BasePage { protected SelenideElement keyTextArea = $x("//div[@id='key']/textarea"); protected SelenideElement valueTextArea = $x("//div[@id='content']/textarea"); protected SelenideElement headersTextArea = $x("//div[@id='headers']/textarea"); protected SelenideElement submitProduceMessageBtn = headersTextArea.$x("../../../..//button[@type='submit']"); protected SelenideElement partitionDdl = $x("//ul[@name='partition']"); protected SelenideElement keySerdeDdl = $x("//ul[@name='keySerde']"); protected SelenideElement contentSerdeDdl = $x("//ul[@name='valueSerde']"); @Step public ProduceMessagePanel waitUntilScreenReady() { waitUntilSpinnerDisappear(); Arrays.asList(partitionDdl, keySerdeDdl, contentSerdeDdl).forEach(element -> element.shouldBe(Condition.visible)); return this; } @Step public ProduceMessagePanel setKeyField(String value) { clearByKeyboard(keyTextArea); keyTextArea.setValue(value); return this; } @Step public ProduceMessagePanel setValueFiled(String value) { clearByKeyboard(valueTextArea); valueTextArea.setValue(value); return this; } @Step public ProduceMessagePanel setHeadersFld(String value) { headersTextArea.setValue(value); return this; } @Step public ProduceMessagePanel submitProduceMessage() { clickByActions(submitProduceMessageBtn); submitProduceMessageBtn.shouldBe(Condition.disappear); refresh(); return this; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/TopicCreateEditForm.java ================================================ package com.provectus.kafka.ui.pages.topics; import static com.codeborne.selenide.Selenide.$; import static com.codeborne.selenide.Selenide.$$; import static com.codeborne.selenide.Selenide.$x; import static org.openqa.selenium.By.id; import com.codeborne.selenide.ClickOptions; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.ElementsCollection; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue; import com.provectus.kafka.ui.pages.topics.enums.CustomParameterType; import com.provectus.kafka.ui.pages.topics.enums.MaxSizeOnDisk; import com.provectus.kafka.ui.pages.topics.enums.TimeToRetain; import io.qameta.allure.Step; public class TopicCreateEditForm extends BasePage { private static final String RETENTION_BYTES = "retentionBytes"; protected SelenideElement timeToRetainField = $x("//input[@id='timeToRetain']"); protected SelenideElement partitionsField = $x("//input[@name='partitions']"); protected SelenideElement nameField = $(id("topicFormName")); protected SelenideElement maxMessageBytesField = $x("//input[@name='maxMessageBytes']"); protected SelenideElement minInSyncReplicasField = $x("//input[@name='minInSyncReplicas']"); protected SelenideElement cleanUpPolicyDdl = $x("//ul[@id='topicFormCleanupPolicy']"); protected SelenideElement maxSizeOnDiscDdl = $x("//ul[@id='topicFormRetentionBytes']"); protected SelenideElement customParameterDdl = $x("//ul[contains(@name,'customParams')]"); protected SelenideElement deleteCustomParameterBtn = $x("//span[contains(@title,'Delete customParam')]"); protected SelenideElement addCustomParameterTypeBtn = $x("//button[contains(text(),'Add Custom Parameter')]"); protected SelenideElement customParameterValueField = $x("//input[@placeholder='Value']"); protected SelenideElement validationCustomParameterValueMsg = $x("//p[contains(text(),'Value is required')]"); protected String ddlElementLocator = "//li[@value='%s']"; protected String btnTimeToRetainLocator = "//button[@class][text()='%s']"; @Step public TopicCreateEditForm waitUntilScreenReady() { waitUntilSpinnerDisappear(); nameField.shouldBe(Condition.visible); return this; } public boolean isCreateTopicButtonEnabled() { return isEnabled(submitBtn); } public boolean isDeleteCustomParameterButtonEnabled() { return isEnabled(deleteCustomParameterBtn); } public boolean isNameFieldEnabled() { return isEnabled(nameField); } @Step public TopicCreateEditForm setTopicName(String topicName) { sendKeysAfterClear(nameField, topicName); return this; } @Step public TopicCreateEditForm setMinInsyncReplicas(Integer minInsyncReplicas) { minInSyncReplicasField.setValue(minInsyncReplicas.toString()); return this; } @Step public TopicCreateEditForm setTimeToRetainDataInMs(Long ms) { timeToRetainField.setValue(ms.toString()); return this; } @Step public TopicCreateEditForm setTimeToRetainDataInMs(String ms) { timeToRetainField.setValue(ms); return this; } @Step public TopicCreateEditForm setMaxSizeOnDiskInGB(MaxSizeOnDisk maxSizeOnDisk) { maxSizeOnDiscDdl.shouldBe(Condition.visible).click(); $x(String.format(ddlElementLocator, maxSizeOnDisk.getOptionValue())).shouldBe(Condition.visible).click(); return this; } @Step public TopicCreateEditForm clickAddCustomParameterTypeButton() { addCustomParameterTypeBtn.click(); return this; } @Step public TopicCreateEditForm openCustomParameterTypeDdl() { customParameterDdl.shouldBe(Condition.visible).click(); ddlOptions.shouldHave(CollectionCondition.sizeGreaterThan(0)); return this; } @Step public ElementsCollection getAllDdlOptions() { return getDdlOptions(); } @Step public TopicCreateEditForm setCustomParameterType(CustomParameterType customParameterType) { openCustomParameterTypeDdl(); $x(String.format(ddlElementLocator, customParameterType.getOptionValue())).shouldBe(Condition.visible).click(); return this; } @Step public TopicCreateEditForm clearCustomParameterValue() { clearByKeyboard(customParameterValueField); return this; } @Step public TopicCreateEditForm setNumberOfPartitions(int partitions) { partitionsField.shouldBe(Condition.enabled).clear(); partitionsField.sendKeys(String.valueOf(partitions)); return this; } @Step public TopicCreateEditForm setTimeToRetainDataByButtons(TimeToRetain timeToRetain) { $x(String.format(btnTimeToRetainLocator, timeToRetain.getButton())).shouldBe(Condition.enabled).click(); return this; } @Step public TopicCreateEditForm selectCleanupPolicy(CleanupPolicyValue cleanupPolicyOptionValue) { cleanUpPolicyDdl.shouldBe(Condition.visible).click(); $x(String.format(ddlElementLocator, cleanupPolicyOptionValue.getOptionValue())).shouldBe(Condition.visible).click(); return this; } @Step public TopicCreateEditForm selectRetentionBytes(String visibleValue) { return selectFromDropDownByVisibleText(RETENTION_BYTES, visibleValue); } @Step public TopicCreateEditForm selectRetentionBytes(Long optionValue) { return selectFromDropDownByOptionValue(RETENTION_BYTES, optionValue.toString()); } @Step public TopicCreateEditForm clickSaveTopicBtn() { clickSubmitBtn(); return this; } @Step public TopicCreateEditForm addCustomParameter(String customParameterName, String customParameterValue) { ElementsCollection customParametersElements = $$("ul[role=listbox][name^=customParams][name$=name]"); KafkaUiSelectElement kafkaUiSelectElement = null; if (customParametersElements.size() == 1) { if ("Select".equals(customParametersElements.first().getText())) { kafkaUiSelectElement = new KafkaUiSelectElement(customParametersElements.first()); } } else { $$("button") .find(Condition.exactText("Add Custom Parameter")) .click(); customParametersElements = $$("ul[role=listbox][name^=customParams][name$=name]"); kafkaUiSelectElement = new KafkaUiSelectElement(customParametersElements.last()); } if (kafkaUiSelectElement != null) { kafkaUiSelectElement.selectByVisibleText(customParameterName); } $(String.format("input[name=\"customParams.%d.value\"]", customParametersElements.size() - 1)) .setValue(customParameterValue); return this; } @Step public TopicCreateEditForm updateCustomParameter(String customParameterName, String customParameterValue) { SelenideElement selenideElement = $$("ul[role=listbox][name^=customParams][name$=name]") .find(Condition.exactText(customParameterName)); String name = selenideElement.getAttribute("name"); if (name != null) { name = name.substring(0, name.lastIndexOf(".")); } $(String.format("input[name^=%s]", name)).setValue(customParameterValue); return this; } @Step public String getCleanupPolicy() { return new KafkaUiSelectElement("cleanupPolicy").getCurrentValue(); } @Step public String getTimeToRetain() { return timeToRetainField.getValue(); } @Step public String getMaxSizeOnDisk() { return new KafkaUiSelectElement(RETENTION_BYTES).getCurrentValue(); } @Step public String getMaxMessageBytes() { return maxMessageBytesField.getValue(); } @Step public TopicCreateEditForm setMaxMessageBytes(Long bytes) { maxMessageBytesField.setValue(bytes.toString()); return this; } @Step public TopicCreateEditForm setMaxMessageBytes(String bytes) { return setMaxMessageBytes(Long.parseLong(bytes)); } @Step public boolean isValidationMessageCustomParameterValueVisible() { return isVisible(validationCustomParameterValueMsg); } @Step public String getCustomParameterValue() { return customParameterValueField.getValue(); } private TopicCreateEditForm selectFromDropDownByOptionValue(String dropDownElementName, String optionValue) { KafkaUiSelectElement select = new KafkaUiSelectElement(dropDownElementName); select.selectByOptionValue(optionValue); return this; } private TopicCreateEditForm selectFromDropDownByVisibleText(String dropDownElementName, String visibleText) { KafkaUiSelectElement select = new KafkaUiSelectElement(dropDownElementName); select.selectByVisibleText(visibleText); return this; } private static class KafkaUiSelectElement { private final SelenideElement selectElement; public KafkaUiSelectElement(String selectElementName) { this.selectElement = $("ul[role=listbox][name=" + selectElementName + "]"); } public KafkaUiSelectElement(SelenideElement selectElement) { this.selectElement = selectElement; } public void selectByOptionValue(String optionValue) { selectElement.click(); selectElement .$$x(".//ul/li[@role='option']") .find(Condition.attribute("value", optionValue)) .click(ClickOptions.usingJavaScript()); } public void selectByVisibleText(String visibleText) { selectElement.click(); selectElement .$$("ul>li[role=option]") .find(Condition.exactText(visibleText)) .click(); } public String getCurrentValue() { return selectElement.$("li").getText(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/TopicDetails.java ================================================ package com.provectus.kafka.ui.pages.topics; import static com.codeborne.selenide.Selenide.$$x; import static com.codeborne.selenide.Selenide.$x; import static com.codeborne.selenide.Selenide.sleep; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.OVERVIEW; import static org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils.nextInt; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.ElementsCollection; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.YearMonth; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; public class TopicDetails extends BasePage { protected SelenideElement clearMessagesBtn = $x(("//div[contains(text(), 'Clear messages')]")); protected SelenideElement recreateTopicBtn = $x("//div[text()='Recreate Topic']"); protected SelenideElement messageAmountCell = $x("//tbody/tr/td[5]"); protected SelenideElement overviewTab = $x("//a[contains(text(),'Overview')]"); protected SelenideElement messagesTab = $x("//a[contains(text(),'Messages')]"); protected SelenideElement seekTypeDdl = $x("//ul[@id='selectSeekType']//li"); protected SelenideElement seekTypeField = $x("//label[text()='Seek Type']//..//div/input"); protected SelenideElement addFiltersBtn = $x("//button[text()='Add Filters']"); protected SelenideElement savedFiltersLink = $x("//div[text()='Saved Filters']"); protected SelenideElement addFilterCodeModalTitle = $x("//label[text()='Filter code']"); protected SelenideElement addFilterCodeEditor = $x("//div[@id='ace-editor']"); protected SelenideElement addFilterCodeTextarea = $x("//div[@id='ace-editor']//textarea"); protected SelenideElement saveThisFilterCheckBoxAddFilterMdl = $x("//input[@name='saveFilter']"); protected SelenideElement displayNameInputAddFilterMdl = $x("//input[@placeholder='Enter Name']"); protected SelenideElement cancelBtnAddFilterMdl = $x("//button[text()='Cancel']"); protected SelenideElement addFilterBtnAddFilterMdl = $x("//button[text()='Add filter']"); protected SelenideElement saveFilterBtnEditFilterMdl = $x("//button[text()='Save']"); protected SelenideElement addFiltersBtnMessages = $x("//button[text()='Add Filters']"); protected SelenideElement selectFilterBtnAddFilterMdl = $x("//button[text()='Select filter']"); protected SelenideElement editSettingsMenu = $x("//li[@role][contains(text(),'Edit settings')]"); protected SelenideElement removeTopicBtn = $x("//ul[@role='menu']//div[contains(text(),'Remove Topic')]"); protected SelenideElement produceMessageBtn = $x("//div//button[text()='Produce Message']"); protected SelenideElement contentMessageTab = $x("//html//div[@id='root']/div/main//table//p"); protected SelenideElement cleanUpPolicyField = $x("//div[contains(text(),'Clean Up Policy')]/../span/*"); protected SelenideElement partitionsField = $x("//div[contains(text(),'Partitions')]/../span"); protected SelenideElement backToCreateFiltersLink = $x("//div[text()='Back To create filters']"); protected ElementsCollection messageGridItems = $$x("//tbody//tr"); protected SelenideElement actualCalendarDate = $x("//div[@class='react-datepicker__current-month']"); protected SelenideElement previousMonthButton = $x("//button[@aria-label='Previous Month']"); protected SelenideElement nextMonthButton = $x("//button[@aria-label='Next Month']"); protected SelenideElement calendarTimeFld = $x("//input[@placeholder='Time']"); protected String detailsTabLtr = "//nav//a[contains(text(),'%s')]"; protected String dayCellLtr = "//div[@role='option'][contains(text(),'%d')]"; protected String seekFilterDdlLocator = "//ul[@id='selectSeekType']/ul/li[text()='%s']"; protected String savedFilterNameLocator = "//div[@role='savedFilter']/div[contains(text(),'%s')]"; protected String consumerIdLocator = "//a[@title='%s']"; protected String topicHeaderLocator = "//h1[contains(text(),'%s')]"; protected String activeFilterNameLocator = "//div[@data-testid='activeSmartFilter']/div[1][contains(text(),'%s')]"; protected String editActiveFilterBtnLocator = "//div[text()='%s']/../div[@data-testid='editActiveSmartFilterBtn']"; protected String settingsGridValueLocator = "//tbody/tr/td/span[text()='%s']//ancestor::tr/td[2]/span"; @Step public TopicDetails waitUntilScreenReady() { waitUntilSpinnerDisappear(); $x(String.format(detailsTabLtr, OVERVIEW)).shouldBe(Condition.visible); return this; } @Step public TopicDetails openDetailsTab(TopicMenu menu) { $x(String.format(detailsTabLtr, menu.toString())).shouldBe(Condition.enabled).click(); waitUntilSpinnerDisappear(); return this; } @Step public String getSettingsGridValueByKey(String key) { return $x(String.format(settingsGridValueLocator, key)).scrollTo().shouldBe(Condition.visible).getText(); } @Step public TopicDetails openDotMenu() { clickByJavaScript(dotMenuBtn); return this; } @Step public boolean isAlertWithMessageVisible(AlertHeader header, String message) { return isAlertVisible(header, message); } @Step public TopicDetails clickEditSettingsMenu() { editSettingsMenu.shouldBe(Condition.visible).click(); return this; } @Step public boolean isConfirmationMdlVisible() { return isConfirmationModalVisible(); } @Step public TopicDetails clickClearMessagesMenu() { clearMessagesBtn.shouldBe(Condition.visible).click(); return this; } @Step public boolean isClearMessagesMenuEnabled() { return !Objects.requireNonNull(clearMessagesBtn.shouldBe(Condition.visible) .$x("./..").getAttribute("class")) .contains("disabled"); } @Step public TopicDetails clickRecreateTopicMenu() { recreateTopicBtn.shouldBe(Condition.visible).click(); return this; } @Step public String getCleanUpPolicy() { return cleanUpPolicyField.getText(); } @Step public int getPartitions() { return Integer.parseInt(partitionsField.getText().trim()); } @Step public boolean isTopicHeaderVisible(String topicName) { return isVisible($x(String.format(topicHeaderLocator, topicName))); } @Step public TopicDetails clickDeleteTopicMenu() { removeTopicBtn.shouldBe(Condition.visible).click(); return this; } @Step public TopicDetails clickConfirmBtnMdl() { clickConfirmButton(); return this; } @Step public TopicDetails clickProduceMessageBtn() { clickByJavaScript(produceMessageBtn); return this; } @Step public TopicDetails selectSeekTypeDdlMessagesTab(String seekTypeName) { seekTypeDdl.shouldBe(Condition.enabled).click(); $x(String.format(seekFilterDdlLocator, seekTypeName)).shouldBe(Condition.visible).click(); return this; } @Step public TopicDetails setSeekTypeValueFldMessagesTab(String seekTypeValue) { seekTypeField.shouldBe(Condition.enabled).sendKeys(seekTypeValue); return this; } @Step public TopicDetails clickSubmitFiltersBtnMessagesTab() { clickByJavaScript(submitBtn); waitUntilSpinnerDisappear(); return this; } @Step public TopicDetails clickMessagesAddFiltersBtn() { addFiltersBtn.shouldBe(Condition.enabled).click(); return this; } @Step public TopicDetails clickEditActiveFilterBtn(String filterName) { $x(String.format(editActiveFilterBtnLocator, filterName)) .shouldBe(Condition.enabled).click(); return this; } @Step public TopicDetails clickNextButton() { clickNextBtn(); waitUntilSpinnerDisappear(); return this; } @Step public TopicDetails openSavedFiltersListMdl() { savedFiltersLink.shouldBe(Condition.enabled).click(); backToCreateFiltersLink.shouldBe(Condition.visible); return this; } @Step public boolean isFilterVisibleAtSavedFiltersMdl(String filterName) { return isVisible($x(String.format(savedFilterNameLocator, filterName))); } @Step public TopicDetails selectFilterAtSavedFiltersMdl(String filterName) { $x(String.format(savedFilterNameLocator, filterName)).shouldBe(Condition.enabled).click(); return this; } @Step public TopicDetails clickSelectFilterBtnAtSavedFiltersMdl() { selectFilterBtnAddFilterMdl.shouldBe(Condition.enabled).click(); addFilterCodeModalTitle.shouldBe(Condition.disappear); return this; } @Step public TopicDetails waitUntilAddFiltersMdlVisible() { addFilterCodeModalTitle.shouldBe(Condition.visible); return this; } @Step public TopicDetails setFilterCodeFldAddFilterMdl(String filterCode) { addFilterCodeTextarea.shouldBe(Condition.enabled).setValue(filterCode); return this; } @Step public String getFilterCodeValue() { addFilterCodeEditor.shouldBe(Condition.enabled).click(); String value = addFilterCodeTextarea.getValue(); if (value == null) { return null; } else { return value.substring(0, value.length() - 2); } } @Step public String getFilterNameValue() { return displayNameInputAddFilterMdl.shouldBe(Condition.enabled).getValue(); } @Step public TopicDetails selectSaveThisFilterCheckboxMdl(boolean select) { selectElement(saveThisFilterCheckBoxAddFilterMdl, select); return this; } @Step public boolean isSaveThisFilterCheckBoxSelected() { return isSelected(saveThisFilterCheckBoxAddFilterMdl); } @Step public TopicDetails setDisplayNameFldAddFilterMdl(String displayName) { displayNameInputAddFilterMdl.shouldBe(Condition.enabled).setValue(displayName); return this; } @Step public TopicDetails clickAddFilterBtnAndCloseMdl(boolean closeModal) { addFilterBtnAddFilterMdl.shouldBe(Condition.enabled).click(); if (closeModal) { addFilterCodeModalTitle.shouldBe(Condition.hidden); } else { addFilterCodeModalTitle.shouldBe(Condition.visible); } return this; } @Step public TopicDetails clickSaveFilterBtnAndCloseMdl(boolean closeModal) { saveFilterBtnEditFilterMdl.shouldBe(Condition.enabled).click(); if (closeModal) { addFilterCodeModalTitle.shouldBe(Condition.hidden); } else { addFilterCodeModalTitle.shouldBe(Condition.visible); } return this; } @Step public boolean isAddFilterBtnAddFilterMdlEnabled() { return isEnabled(addFilterBtnAddFilterMdl); } @Step public boolean isBackButtonEnabled() { return isEnabled(backBtn); } @Step public boolean isNextButtonEnabled() { return isEnabled(nextBtn); } @Step public boolean isActiveFilterVisible(String filterName) { return isVisible($x(String.format(activeFilterNameLocator, filterName))); } @Step public String getSearchFieldValue() { return searchFld.shouldBe(Condition.visible).getValue(); } public List getAllAddFilterModalVisibleElements() { return Arrays.asList(savedFiltersLink, displayNameInputAddFilterMdl, addFilterBtnAddFilterMdl, cancelBtnAddFilterMdl); } public List getAllAddFilterModalEnabledElements() { return Arrays.asList(displayNameInputAddFilterMdl, cancelBtnAddFilterMdl); } public List getAllAddFilterModalDisabledElements() { return Collections.singletonList(addFilterBtnAddFilterMdl); } @Step public TopicDetails openConsumerGroup(String consumerId) { $x(String.format(consumerIdLocator, consumerId)).click(); return this; } private void selectYear(int expectedYear) { while (getActualCalendarDate().getYear() > expectedYear) { clickByJavaScript(previousMonthButton); sleep(1000); if (LocalTime.now().plusMinutes(3).isBefore(LocalTime.now())) { throw new IllegalArgumentException("Unable to select year"); } } } private void selectMonth(int expectedMonth) { while (getActualCalendarDate().getMonthValue() > expectedMonth) { clickByJavaScript(previousMonthButton); sleep(1000); if (LocalTime.now().plusMinutes(3).isBefore(LocalTime.now())) { throw new IllegalArgumentException("Unable to select month"); } } } private void selectDay(int expectedDay) { Objects.requireNonNull($$x(String.format(dayCellLtr, expectedDay)).stream() .filter(day -> !Objects.requireNonNull(day.getAttribute("class")).contains("outside-month")) .findFirst().orElseThrow()).shouldBe(Condition.enabled).click(); } private void setTime(LocalDateTime dateTime) { calendarTimeFld.shouldBe(Condition.enabled) .sendKeys(String.valueOf(dateTime.getHour()), String.valueOf(dateTime.getMinute())); } @Step public TopicDetails selectDateAndTimeByCalendar(LocalDateTime dateTime) { setTime(dateTime); selectYear(dateTime.getYear()); selectMonth(dateTime.getMonthValue()); selectDay(dateTime.getDayOfMonth()); return this; } private LocalDate getActualCalendarDate() { String monthAndYearStr = actualCalendarDate.getText().trim(); DateTimeFormatter formatter = new DateTimeFormatterBuilder() .parseCaseInsensitive() .append(DateTimeFormatter.ofPattern("MMMM yyyy")) .toFormatter(Locale.ENGLISH); YearMonth yearMonth = formatter.parse(monthAndYearStr, YearMonth::from); return yearMonth.atDay(1); } @Step public TopicDetails openCalendarSeekType() { seekTypeField.shouldBe(Condition.enabled).click(); actualCalendarDate.shouldBe(Condition.visible); return this; } @Step public int getMessageCountAmount() { return Integer.parseInt(messageAmountCell.getText().trim()); } private List initItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new TopicDetails.MessageGridItem(item))); return gridItemList; } @Step public TopicDetails.MessageGridItem getMessageByOffset(int offset) { return initItems().stream() .filter(e -> e.getOffset() == offset) .findFirst().orElseThrow(); } @Step public TopicDetails.MessageGridItem getMessageByKey(String key) { return initItems().stream() .filter(e -> e.getKey().equals(key)) .findFirst().orElseThrow(); } @Step public List getAllMessages() { return initItems(); } @Step public TopicDetails.MessageGridItem getRandomMessage() { return getMessageByOffset(nextInt(0, initItems().size() - 1)); } public enum TopicMenu { OVERVIEW("Overview"), MESSAGES("Messages"), CONSUMERS("Consumers"), SETTINGS("Settings"); private final String value; TopicMenu(String value) { this.value = value; } public String toString() { return value; } } public static class MessageGridItem extends BasePage { private final SelenideElement element; private MessageGridItem(SelenideElement element) { this.element = element; } @Step public MessageGridItem clickExpand() { clickByJavaScript(element.$x("./td[1]/span")); return this; } private SelenideElement getOffsetElm() { return element.$x("./td[2]"); } @Step public int getOffset() { return Integer.parseInt(getOffsetElm().getText().trim()); } @Step public int getPartition() { return Integer.parseInt(element.$x("./td[3]").getText().trim()); } @Step public LocalDateTime getTimestamp() { String timestampValue = element.$x("./td[4]/div").getText().trim(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("M/d/yyyy, HH:mm:ss"); return LocalDateTime.parse(timestampValue, formatter); } @Step public String getKey() { return element.$x("./td[5]").getText().trim(); } @Step public String getValue() { return element.$x("./td[6]").getAttribute("title"); } @Step public MessageGridItem openDotMenu() { getOffsetElm().hover(); element.$x("./td[7]/div/button[@aria-label='Dropdown Toggle']") .shouldBe(Condition.visible).click(); return this; } @Step public MessageGridItem clickCopyToClipBoard() { clickByJavaScript(element.$x("./td[7]//li[text() = 'Copy to clipboard']") .shouldBe(Condition.visible)); return this; } @Step public MessageGridItem clickSaveAsFile() { clickByJavaScript(element.$x("./td[7]//li[text() = 'Save as a file']") .shouldBe(Condition.visible)); return this; } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/TopicSettingsTab.java ================================================ package com.provectus.kafka.ui.pages.topics; import static com.codeborne.selenide.Selenide.$x; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.ArrayList; import java.util.List; public class TopicSettingsTab extends BasePage { protected SelenideElement defaultValueColumnHeaderLocator = $x("//div[text() = 'Default Value']"); @Step public TopicSettingsTab waitUntilScreenReady() { waitUntilSpinnerDisappear(); defaultValueColumnHeaderLocator.shouldBe(Condition.visible); return this; } private List initGridItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new SettingsGridItem(item))); return gridItemList; } private TopicSettingsTab.SettingsGridItem getItemByKey(String key) { return initGridItems().stream() .filter(e -> e.getKey().equals(key)) .findFirst().orElseThrow(); } @Step public String getValueByKey(String key) { return getItemByKey(key).getValue(); } public static class SettingsGridItem extends BasePage { private final SelenideElement element; public SettingsGridItem(SelenideElement element) { this.element = element; } @Step public String getKey() { return element.$x("./td[1]/span").getText().trim(); } @Step public String getValue() { return element.$x("./td[2]/span").getText().trim(); } @Step public String getDefaultValue() { return element.$x("./td[3]/span").getText().trim(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/TopicsList.java ================================================ package com.provectus.kafka.ui.pages.topics; import static com.codeborne.selenide.Condition.visible; import static com.codeborne.selenide.Selenide.$x; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.TOPICS; import com.codeborne.selenide.CollectionCondition; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.pages.BasePage; import io.qameta.allure.Step; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; public class TopicsList extends BasePage { protected SelenideElement addTopicBtn = $x("//button[normalize-space(text()) ='Add a Topic']"); protected SelenideElement searchField = $x("//input[@placeholder='Search by Topic Name']"); protected SelenideElement showInternalRadioBtn = $x("//input[@name='ShowInternalTopics']"); protected SelenideElement deleteSelectedTopicsBtn = $x("//button[text()='Delete selected topics']"); protected SelenideElement copySelectedTopicBtn = $x("//button[text()='Copy selected topic']"); protected SelenideElement purgeMessagesOfSelectedTopicsBtn = $x("//button[text()='Purge messages of selected topics']"); protected SelenideElement clearMessagesBtn = $x("//ul[contains(@class ,'open')]//div[text()='Clear Messages']"); protected SelenideElement recreateTopicBtn = $x("//ul[contains(@class ,'open')]//div[text()='Recreate Topic']"); protected SelenideElement removeTopicBtn = $x("//ul[contains(@class ,'open')]//div[text()='Remove Topic']"); @Step public TopicsList waitUntilScreenReady() { waitUntilSpinnerDisappear(); getPageTitleFromHeader(TOPICS).shouldBe(visible); return this; } @Step public TopicsList clickAddTopicBtn() { clickByJavaScript(addTopicBtn); return this; } @Step public boolean isTopicVisible(String topicName) { tableGrid.shouldBe(visible); return isVisible(getTableElement(topicName)); } @Step public boolean isShowInternalRadioBtnSelected() { return isSelected(showInternalRadioBtn); } @Step public TopicsList setShowInternalRadioButton(boolean select) { if (select) { if (!showInternalRadioBtn.isSelected()) { clickByJavaScript(showInternalRadioBtn); waitUntilSpinnerDisappear(1); } } else { if (showInternalRadioBtn.isSelected()) { clickByJavaScript(showInternalRadioBtn); waitUntilSpinnerDisappear(1); } } return this; } @Step public TopicsList goToLastPage() { if (nextBtn.exists()) { while (nextBtn.isEnabled()) { clickNextBtn(); waitUntilSpinnerDisappear(1); } } return this; } @Step public TopicsList openTopic(String topicName) { getTopicItem(topicName).openItem(); return this; } @Step public TopicsList openDotMenuByTopicName(String topicName) { getTopicItem(topicName).openDotMenu(); return this; } @Step public boolean isCopySelectedTopicBtnEnabled() { return isEnabled(copySelectedTopicBtn); } @Step public List getActionButtons() { return Stream.of(deleteSelectedTopicsBtn, copySelectedTopicBtn, purgeMessagesOfSelectedTopicsBtn) .collect(Collectors.toList()); } @Step public TopicsList clickCopySelectedTopicBtn() { copySelectedTopicBtn.shouldBe(Condition.enabled).click(); return this; } @Step public TopicsList clickPurgeMessagesOfSelectedTopicsBtn() { purgeMessagesOfSelectedTopicsBtn.shouldBe(Condition.enabled).click(); return this; } @Step public TopicsList clickClearMessagesBtn() { clickByJavaScript(clearMessagesBtn.shouldBe(visible)); return this; } @Step public TopicsList clickRecreateTopicBtn() { clickByJavaScript(recreateTopicBtn.shouldBe(visible)); return this; } @Step public TopicsList clickRemoveTopicBtn() { clickByJavaScript(removeTopicBtn.shouldBe(visible)); return this; } @Step public TopicsList clickConfirmBtnMdl() { clickConfirmButton(); return this; } @Step public TopicsList clickCancelBtnMdl() { clickCancelButton(); return this; } @Step public boolean isConfirmationMdlVisible() { return isConfirmationModalVisible(); } @Step public boolean isAlertWithMessageVisible(AlertHeader header, String message) { return isAlertVisible(header, message); } private List getVisibleColumnHeaders() { return Stream.of("Replication Factor", "Number of messages", "Topic Name", "Partitions", "Out of sync replicas", "Size") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } private List getEnabledColumnHeaders() { return Stream.of("Topic Name", "Partitions", "Out of sync replicas", "Size") .map(name -> $x(String.format(columnHeaderLocator, name))) .collect(Collectors.toList()); } @Step public List getAllVisibleElements() { List visibleElements = new ArrayList<>(getVisibleColumnHeaders()); visibleElements.addAll(Arrays.asList(searchField, addTopicBtn, tableGrid)); visibleElements.addAll(getActionButtons()); return visibleElements; } @Step public List getAllEnabledElements() { List enabledElements = new ArrayList<>(getEnabledColumnHeaders()); enabledElements.addAll(Arrays.asList(searchField, showInternalRadioBtn, addTopicBtn)); return enabledElements; } private List initGridItems() { List gridItemList = new ArrayList<>(); gridItems.shouldHave(CollectionCondition.sizeGreaterThan(0)) .forEach(item -> gridItemList.add(new TopicGridItem(item))); return gridItemList; } @Step public TopicGridItem getTopicItem(String name) { TopicGridItem topicGridItem = initGridItems().stream() .filter(e -> e.getName().equals(name)) .findFirst().orElse(null); if (topicGridItem == null) { searchItem(name); topicGridItem = initGridItems().stream() .filter(e -> e.getName().equals(name)) .findFirst().orElseThrow(); } return topicGridItem; } @Step public TopicGridItem getAnyNonInternalTopic() { return getNonInternalTopics().stream() .findAny().orElseThrow(); } @Step public List getNonInternalTopics() { return initGridItems().stream() .filter(e -> !e.isInternal()) .collect(Collectors.toList()); } @Step public List getInternalTopics() { return initGridItems().stream() .filter(TopicGridItem::isInternal) .collect(Collectors.toList()); } public static class TopicGridItem extends BasePage { private final SelenideElement element; public TopicGridItem(SelenideElement element) { this.element = element; } @Step public TopicsList selectItem(boolean select) { selectElement(element.$x("./td[1]/input"), select); return new TopicsList(); } private SelenideElement getNameElm() { return element.$x("./td[2]"); } @Step public boolean isInternal() { boolean internal = false; try { internal = getNameElm().$x("./a/span").isDisplayed(); } catch (Throwable ignored) { } return internal; } @Step public String getName() { return getNameElm().$x("./a").getAttribute("title"); } @Step public void openItem() { getNameElm().click(); } @Step public int getPartition() { return Integer.parseInt(element.$x("./td[3]").getText().trim()); } @Step public int getOutOfSyncReplicas() { return Integer.parseInt(element.$x("./td[4]").getText().trim()); } @Step public int getReplicationFactor() { return Integer.parseInt(element.$x("./td[5]").getText().trim()); } @Step public int getNumberOfMessages() { return Integer.parseInt(element.$x("./td[6]").getText().trim()); } @Step public int getSize() { return Integer.parseInt(element.$x("./td[7]").getText().trim()); } @Step public void openDotMenu() { element.$x("./td[8]//button").click(); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/enums/CleanupPolicyValue.java ================================================ package com.provectus.kafka.ui.pages.topics.enums; public enum CleanupPolicyValue { DELETE("delete", "Delete"), COMPACT("compact", "Compact"), COMPACT_DELETE("compact,delete", "Compact,Delete"); private final String optionValue; private final String visibleText; CleanupPolicyValue(String optionValue, String visibleText) { this.optionValue = optionValue; this.visibleText = visibleText; } public String getOptionValue() { return optionValue; } public String getVisibleText() { return visibleText; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/enums/CustomParameterType.java ================================================ package com.provectus.kafka.ui.pages.topics.enums; public enum CustomParameterType { COMPRESSION_TYPE("compression.type"), DELETE_RETENTION_MS("delete.retention.ms"), FILE_DELETE_DELAY_MS("file.delete.delay.ms"), FLUSH_MESSAGES("flush.messages"), FLUSH_MS("flush.ms"), FOLLOWER_REPLICATION_THROTTLED_REPLICAS("follower.replication.throttled.replicas"), INDEX_INTERVAL_BYTES("index.interval.bytes"), LEADER_REPLICATION_THROTTLED_REPLICAS("leader.replication.throttled.replicas"), MAX_COMPACTION_LAG_MS("max.compaction.lag.ms"), MESSAGE_DOWNCONVERSION_ENABLE("message.downconversion.enable"), MESSAGE_FORMAT_VERSION("message.format.version"), MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS("message.timestamp.difference.max.ms"), MESSAGE_TIMESTAMP_TYPE("message.timestamp.type"), MIN_CLEANABLE_DIRTY_RATIO("min.cleanable.dirty.ratio"), MIN_COMPACTION_LAG_MS("min.compaction.lag.ms"), PREALLOCATE("preallocate"), RETENTION_BYTES("retention.bytes"), SEGMENT_BYTES("segment.bytes"), SEGMENT_INDEX_BYTES("segment.index.bytes"), SEGMENT_JITTER_MS("segment.jitter.ms"), SEGMENT_MS("segment.ms"), UNCLEAN_LEADER_ELECTION_ENABLE("unclean.leader.election.enable"); private final String optionValue; CustomParameterType(String optionValue) { this.optionValue = optionValue; } public String getOptionValue() { return optionValue; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/enums/MaxSizeOnDisk.java ================================================ package com.provectus.kafka.ui.pages.topics.enums; public enum MaxSizeOnDisk { NOT_SET("-1", "Not Set"), SIZE_1_GB("1073741824", "1 GB"), SIZE_10_GB("10737418240", "10 GB"), SIZE_20_GB("21474836480", "20 GB"), SIZE_50_GB("53687091200", "50 GB"); private final String optionValue; private final String visibleText; MaxSizeOnDisk(String optionValue, String visibleText) { this.optionValue = optionValue; this.visibleText = visibleText; } public String getOptionValue() { return optionValue; } public String getVisibleText() { return visibleText; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/pages/topics/enums/TimeToRetain.java ================================================ package com.provectus.kafka.ui.pages.topics.enums; public enum TimeToRetain { BTN_12_HOURS("12 hours", "43200000"), BTN_1_DAY("1 day", "86400000"), BTN_2_DAYS("2 days", "172800000"), BTN_7_DAYS("7 days", "604800000"), BTN_4_WEEKS("4 weeks", "2419200000"); private final String button; private final String value; TimeToRetain(String button, String value) { this.button = button; this.value = value; } public String getButton() { return button; } public String getValue() { return value; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/services/ApiService.java ================================================ package com.provectus.kafka.ui.services; import static com.codeborne.selenide.Selenide.sleep; import static com.provectus.kafka.ui.utilities.FileUtils.fileToString; import com.fasterxml.jackson.databind.ObjectMapper; import com.provectus.kafka.ui.api.ApiClient; import com.provectus.kafka.ui.api.api.KafkaConnectApi; import com.provectus.kafka.ui.api.api.KsqlApi; import com.provectus.kafka.ui.api.api.MessagesApi; import com.provectus.kafka.ui.api.api.SchemasApi; import com.provectus.kafka.ui.api.api.TopicsApi; import com.provectus.kafka.ui.api.model.CreateTopicMessage; import com.provectus.kafka.ui.api.model.KsqlCommandV2; import com.provectus.kafka.ui.api.model.KsqlCommandV2Response; import com.provectus.kafka.ui.api.model.KsqlResponse; import com.provectus.kafka.ui.api.model.NewConnector; import com.provectus.kafka.ui.api.model.NewSchemaSubject; import com.provectus.kafka.ui.api.model.TopicCreation; import com.provectus.kafka.ui.models.Connector; import com.provectus.kafka.ui.models.Schema; import com.provectus.kafka.ui.models.Topic; import com.provectus.kafka.ui.pages.ksqldb.models.Stream; import com.provectus.kafka.ui.pages.ksqldb.models.Table; import com.provectus.kafka.ui.settings.BaseSource; import io.qameta.allure.Step; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.web.reactive.function.client.WebClientResponseException; @Slf4j public class ApiService extends BaseSource { private final ApiClient apiClient = new ApiClient().setBasePath(BASE_API_URL); @SneakyThrows private TopicsApi topicApi() { return new TopicsApi(apiClient); } @SneakyThrows private SchemasApi schemaApi() { return new SchemasApi(apiClient); } @SneakyThrows private KafkaConnectApi connectorApi() { return new KafkaConnectApi(apiClient); } @SneakyThrows private MessagesApi messageApi() { return new MessagesApi(apiClient); } @SneakyThrows private KsqlApi ksqlApi() { return new KsqlApi(apiClient); } @SneakyThrows private void createTopic(String clusterName, String topicName) { TopicCreation topic = new TopicCreation(); topic.setName(topicName); topic.setPartitions(1); topic.setReplicationFactor(1); try { topicApi().createTopic(clusterName, topic).block(); sleep(2000); } catch (WebClientResponseException ex) { ex.printStackTrace(); } } @Step public ApiService createTopic(Topic topic) { createTopic(CLUSTER_NAME, topic.getName()); return this; } @SneakyThrows private void deleteTopic(String clusterName, String topicName) { try { topicApi().deleteTopic(clusterName, topicName).block(); } catch (WebClientResponseException ignored) { } } @Step public ApiService deleteTopic(String topicName) { deleteTopic(CLUSTER_NAME, topicName); return this; } @SneakyThrows private void createSchema(String clusterName, Schema schema) { NewSchemaSubject schemaSubject = new NewSchemaSubject(); schemaSubject.setSubject(schema.getName()); schemaSubject.setSchema(fileToString(schema.getValuePath())); schemaSubject.setSchemaType(schema.getType()); try { schemaApi().createNewSchema(clusterName, schemaSubject).block(); } catch (WebClientResponseException ex) { ex.printStackTrace(); } } @Step public ApiService createSchema(Schema schema) { createSchema(CLUSTER_NAME, schema); return this; } @SneakyThrows private void deleteSchema(String clusterName, String schemaName) { try { schemaApi().deleteSchema(clusterName, schemaName).block(); } catch (WebClientResponseException ignored) { } } @Step public ApiService deleteSchema(String schemaName) { deleteSchema(CLUSTER_NAME, schemaName); return this; } @SneakyThrows private void deleteConnector(String clusterName, String connectName, String connectorName) { try { connectorApi().deleteConnector(clusterName, connectName, connectorName).block(); } catch (WebClientResponseException ignored) { } } @Step public ApiService deleteConnector(String connectName, String connectorName) { deleteConnector(CLUSTER_NAME, connectName, connectorName); return this; } @Step public ApiService deleteConnector(String connectorName) { deleteConnector(CLUSTER_NAME, CONNECT_NAME, connectorName); return this; } @SneakyThrows private void createConnector(String clusterName, String connectName, Connector connector) { NewConnector connectorProperties = new NewConnector(); connectorProperties.setName(connector.getName()); Map configMap = new ObjectMapper().readValue(connector.getConfig(), HashMap.class); connectorProperties.setConfig(configMap); try { connectorApi().deleteConnector(clusterName, connectName, connector.getName()).block(); } catch (WebClientResponseException ignored) { } connectorApi().createConnector(clusterName, connectName, connectorProperties).block(); } @Step public ApiService createConnector(String connectName, Connector connector) { createConnector(CLUSTER_NAME, connectName, connector); return this; } @Step public ApiService createConnector(Connector connector) { createConnector(CLUSTER_NAME, CONNECT_NAME, connector); return this; } @Step public String getFirstConnectName(String clusterName) { return Objects.requireNonNull(connectorApi().getConnects(clusterName).blockFirst()).getName(); } @SneakyThrows private void sendMessage(String clusterName, Topic topic) { CreateTopicMessage createMessage = new CreateTopicMessage(); createMessage.setPartition(0); createMessage.setKeySerde("String"); createMessage.setValueSerde("String"); createMessage.setKey(topic.getMessageKey()); createMessage.setContent(topic.getMessageValue()); try { messageApi().sendTopicMessages(clusterName, topic.getName(), createMessage).block(); } catch (WebClientResponseException ex) { ex.getRawStatusCode(); } } @Step public ApiService sendMessage(Topic topic) { sendMessage(CLUSTER_NAME, topic); return this; } @Step public ApiService createStream(Stream stream) { KsqlCommandV2Response pipeIdStream = ksqlApi() .executeKsql(CLUSTER_NAME, new KsqlCommandV2() .ksql(String.format("CREATE STREAM %s (profileId VARCHAR, latitude DOUBLE, longitude DOUBLE) ", stream.getName()) + String.format("WITH (kafka_topic='%s', value_format='json', partitions=1);", stream.getTopicName()))) .block(); assert pipeIdStream != null; List responseListStream = ksqlApi() .openKsqlResponsePipe(CLUSTER_NAME, pipeIdStream.getPipeId()) .collectList() .block(); assert Objects.requireNonNull(responseListStream).size() != 0; return this; } @Step public ApiService createTables(Table firstTable, Table secondTable) { KsqlCommandV2Response pipeIdTable1 = ksqlApi() .executeKsql(CLUSTER_NAME, new KsqlCommandV2() .ksql(String.format("CREATE TABLE %s AS ", firstTable.getName()) + " SELECT profileId, " + " LATEST_BY_OFFSET(latitude) AS la, " + " LATEST_BY_OFFSET(longitude) AS lo " + String.format(" FROM %s ", firstTable.getStreamName()) + " GROUP BY profileId " + " EMIT CHANGES;")) .block(); assert pipeIdTable1 != null; List responseListTable = ksqlApi() .openKsqlResponsePipe(CLUSTER_NAME, pipeIdTable1.getPipeId()) .collectList() .block(); assert Objects.requireNonNull(responseListTable).size() != 0; KsqlCommandV2Response pipeIdTable2 = ksqlApi() .executeKsql(CLUSTER_NAME, new KsqlCommandV2() .ksql(String.format("CREATE TABLE %s AS ", secondTable.getName()) + " SELECT ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1) AS distanceInMiles, " + " COLLECT_LIST(profileId) AS riders, " + " COUNT(*) AS count " + String.format(" FROM %s ", firstTable.getName()) + " GROUP BY ROUND(GEO_DISTANCE(la, lo, 37.4133, -122.1162), -1);")) .block(); assert pipeIdTable2 != null; List responseListTable2 = ksqlApi() .openKsqlResponsePipe(CLUSTER_NAME, pipeIdTable2.getPipeId()) .collectList() .block(); assert Objects.requireNonNull(responseListTable2).size() != 0; return this; } @Step public ApiService insertInto(Stream stream) { String streamName = stream.getName(); KsqlCommandV2Response pipeIdInsert = ksqlApi() .executeKsql(CLUSTER_NAME, new KsqlCommandV2() .ksql("INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('c2309eec', 37.7877, -122.4205);" + "INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('18f4ea86', 37.3903, -122.0643); " + "INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('4ab5cbad', 37.3952, -122.0813); " + "INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('8b6eae59', 37.3944, -122.0813); " + "INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('4a7c7b41', 37.4049, -122.0822); " + "INSERT INTO " + streamName + " (profileId, latitude, longitude) VALUES ('4ddad000', 37.7857, -122.4011);")) .block(); assert pipeIdInsert != null; List responseListInsert = ksqlApi() .openKsqlResponsePipe(CLUSTER_NAME, pipeIdInsert.getPipeId()) .collectList() .block(); assert Objects.requireNonNull(responseListInsert).size() != 0; return this; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/BaseSource.java ================================================ package com.provectus.kafka.ui.settings; import static com.provectus.kafka.ui.variables.Browser.LOCAL; import com.provectus.kafka.ui.settings.configs.Config; import org.aeonbits.owner.ConfigFactory; public abstract class BaseSource { public static final String CLUSTER_NAME = "local"; public static final String CONNECT_NAME = "first"; private static final String LOCAL_HOST = "localhost"; public static final String REMOTE_URL = String.format("http://%s:4444/wd/hub", LOCAL_HOST); public static final String BASE_API_URL = String.format("http://%s:8080", LOCAL_HOST); private static Config config; public static final String BROWSER = config().browser(); public static final String BASE_HOST = BROWSER.equals(LOCAL) ? LOCAL_HOST : "host.docker.internal"; public static final String BASE_UI_URL = String.format("http://%s:8080", BASE_HOST); public static final String SUITE_NAME = config().suite(); private static Config config() { if (config == null) { config = ConfigFactory.create(Config.class, System.getProperties()); } return config; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/configs/Config.java ================================================ package com.provectus.kafka.ui.settings.configs; public interface Config extends Profiles { } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/configs/Profiles.java ================================================ package com.provectus.kafka.ui.settings.configs; import static com.provectus.kafka.ui.variables.Browser.CONTAINER; import static com.provectus.kafka.ui.variables.Suite.CUSTOM; import org.aeonbits.owner.Config; public interface Profiles extends Config { @Key("browser") @DefaultValue(CONTAINER) String browser(); @Key("suite") @DefaultValue(CUSTOM) String suite(); } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/drivers/WebDriver.java ================================================ package com.provectus.kafka.ui.settings.drivers; import static com.codeborne.selenide.Selenide.clearBrowserCookies; import static com.codeborne.selenide.Selenide.clearBrowserLocalStorage; import static com.codeborne.selenide.Selenide.refresh; import static com.provectus.kafka.ui.settings.BaseSource.BROWSER; import static com.provectus.kafka.ui.settings.BaseSource.REMOTE_URL; import static com.provectus.kafka.ui.variables.Browser.CONTAINER; import static com.provectus.kafka.ui.variables.Browser.LOCAL; import com.codeborne.selenide.Configuration; import com.codeborne.selenide.Selenide; import com.codeborne.selenide.WebDriverRunner; import com.codeborne.selenide.logevents.SelenideLogger; import io.qameta.allure.Step; import io.qameta.allure.selenide.AllureSelenide; import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.openqa.selenium.chrome.ChromeOptions; @Slf4j public abstract class WebDriver { @Step public static void browserSetup() { Configuration.headless = false; Configuration.browser = "chrome"; Configuration.browserSize = "1920x1080"; Configuration.screenshots = true; Configuration.savePageSource = false; Configuration.pageLoadTimeout = 120000; ChromeOptions chromeOptions = new ChromeOptions() .addArguments("--no-sandbox") .addArguments("--verbose") .addArguments("--remote-allow-origins=*") .addArguments("--disable-dev-shm-usage") .addArguments("--disable-gpu") .addArguments("--lang=en_US"); switch (BROWSER) { case (LOCAL) -> Configuration.browserCapabilities = chromeOptions; case (CONTAINER) -> { Configuration.remote = REMOTE_URL; Configuration.remoteConnectionTimeout = 180000; Map selenoidOptions = new HashMap<>(); selenoidOptions.put("enableVNC", true); selenoidOptions.put("enableVideo", false); chromeOptions.setCapability("selenoid:options", selenoidOptions); Configuration.browserCapabilities = chromeOptions; } default -> throw new IllegalStateException("Unexpected value: " + BROWSER); } } private static org.openqa.selenium.WebDriver getWebDriver() { try { return WebDriverRunner.getWebDriver(); } catch (IllegalStateException ex) { browserSetup(); Selenide.open(); return WebDriverRunner.getWebDriver(); } } @Step public static void openUrl(String url) { org.openqa.selenium.WebDriver driver = getWebDriver(); if (!driver.getCurrentUrl().equals(url)) { driver.get(url); } } @Step public static void browserInit() { getWebDriver(); } @Step public static void browserClear() { clearBrowserLocalStorage(); clearBrowserCookies(); refresh(); } @Step public static void browserQuit() { org.openqa.selenium.WebDriver driver = null; try { driver = WebDriverRunner.getWebDriver(); } catch (Throwable ignored) { } if (driver != null) { driver.quit(); } } @Step public static void loggerSetup() { SelenideLogger.addListener("AllureSelenide", new AllureSelenide() .screenshots(true) .savePageSource(false)); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/listeners/AllureListener.java ================================================ package com.provectus.kafka.ui.settings.listeners; import static java.nio.file.Files.newInputStream; import com.codeborne.selenide.Screenshots; import io.qameta.allure.Allure; import io.qameta.allure.testng.AllureTestNg; import java.io.File; import java.io.IOException; import lombok.extern.slf4j.Slf4j; import org.testng.ITestListener; import org.testng.ITestResult; @Slf4j public class AllureListener extends AllureTestNg implements ITestListener { private void takeScreenshot() { File screenshot = Screenshots.takeScreenShotAsFile(); try { if (screenshot != null) { Allure.addAttachment(screenshot.getName(), newInputStream(screenshot.toPath())); } else { log.warn("Unable to take screenshot"); } } catch (IOException e) { throw new RuntimeException(e); } } @Override public void onTestFailure(ITestResult result) { takeScreenshot(); } @Override public void onTestSkipped(ITestResult result) { takeScreenshot(); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/listeners/LoggerListener.java ================================================ package com.provectus.kafka.ui.settings.listeners; import lombok.extern.slf4j.Slf4j; import org.testng.ITestResult; import org.testng.TestListenerAdapter; @Slf4j public class LoggerListener extends TestListenerAdapter { @Override public void onTestStart(final ITestResult testResult) { log.info(String.format("\n------------------------------------------------------------------------ " + "\nTEST STARTED: %s.%s \n------------------------------------------------------------------------ \n", testResult.getInstanceName(), testResult.getName())); } @Override public void onTestSuccess(final ITestResult testResult) { log.info(String.format("\n------------------------------------------------------------------------ " + "\nTEST PASSED: %s.%s \n------------------------------------------------------------------------ \n", testResult.getInstanceName(), testResult.getName())); } @Override public void onTestFailure(final ITestResult testResult) { log.info(String.format("\n------------------------------------------------------------------------ " + "\nTEST FAILED: %s.%s \n------------------------------------------------------------------------ \n", testResult.getInstanceName(), testResult.getName())); } @Override public void onTestSkipped(final ITestResult testResult) { log.info(String.format("\n------------------------------------------------------------------------ " + "\nTEST SKIPPED: %s.%s \n------------------------------------------------------------------------ \n", testResult.getInstanceName(), testResult.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/listeners/QaseCreateListener.java ================================================ package com.provectus.kafka.ui.settings.listeners; import static io.qase.api.utils.IntegrationUtils.getCaseTitle; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import com.provectus.kafka.ui.utilities.qase.annotations.Status; import com.provectus.kafka.ui.utilities.qase.annotations.Suite; import io.qase.api.QaseClient; import io.qase.api.StepStorage; import io.qase.api.annotation.QaseId; import io.qase.client.ApiClient; import io.qase.client.api.CasesApi; import io.qase.client.model.GetCasesFiltersParameter; import io.qase.client.model.ResultCreateStepsInner; import io.qase.client.model.TestCase; import io.qase.client.model.TestCaseCreate; import io.qase.client.model.TestCaseCreateStepsInner; import io.qase.client.model.TestCaseListResponse; import io.qase.client.model.TestCaseListResponseAllOfResult; import java.lang.reflect.Method; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.testng.Assert; import org.testng.ITestListener; import org.testng.ITestResult; import org.testng.TestListenerAdapter; @Slf4j public class QaseCreateListener extends TestListenerAdapter implements ITestListener { private static final CasesApi QASE_API = getQaseApi(); private static CasesApi getQaseApi() { ApiClient apiClient = QaseClient.getApiClient(); apiClient.setApiKey(System.getProperty("QASEIO_API_TOKEN")); return new CasesApi(apiClient); } private static int getStatus(Method method) { if (method.isAnnotationPresent(Status.class)) { return method.getDeclaredAnnotation(Status.class).status().getValue(); } return 1; } private static int getAutomation(Method method) { if (method.isAnnotationPresent(Automation.class)) { return method.getDeclaredAnnotation(Automation.class).state().getValue(); } return 0; } @SneakyThrows private static HashMap getCaseTitlesAndIdsFromQase() { HashMap cases = new HashMap<>(); boolean getCases = true; int offSet = 0; while (getCases) { getCases = false; TestCaseListResponse response = QASE_API.getCases(System.getProperty("QASE_PROJECT_CODE"), new GetCasesFiltersParameter().status(GetCasesFiltersParameter.SERIALIZED_NAME_STATUS), 100, offSet); TestCaseListResponseAllOfResult result = response.getResult(); Assert.assertNotNull(result); List entities = result.getEntities(); Assert.assertNotNull(entities); if (entities.size() > 0) { for (TestCase testCase : entities) { cases.put(testCase.getId(), testCase.getTitle()); } offSet = offSet + 100; getCases = true; } } return cases; } private static boolean isCaseWithTitleExistInQase(Method method) { HashMap cases = getCaseTitlesAndIdsFromQase(); String title = getCaseTitle(method); if (cases.containsValue(title)) { for (Map.Entry map : cases.entrySet()) { if (map.getValue().matches(title)) { long id = map.getKey(); log.warn(String.format("Test case with @QaseTitle='%s' already exists with @QaseId=%d. " + "Please verify @QaseTitle annotation", title, id)); return true; } } } return false; } @Override @SneakyThrows public void onTestSuccess(final ITestResult testResult) { Method method = testResult.getMethod() .getConstructorOrMethod() .getMethod(); String title = getCaseTitle(method); if (!method.isAnnotationPresent(QaseId.class)) { if (title != null) { if (!isCaseWithTitleExistInQase(method)) { LinkedList resultSteps = StepStorage.stopSteps(); LinkedList createSteps = new LinkedList<>(); resultSteps.forEach(step -> { TestCaseCreateStepsInner caseStep = new TestCaseCreateStepsInner(); caseStep.setAction(step.getAction()); caseStep.setExpectedResult(step.getExpectedResult()); createSteps.add(caseStep); }); TestCaseCreate newCase = new TestCaseCreate(); newCase.setTitle(title); newCase.setStatus(getStatus(method)); newCase.setAutomation(getAutomation(method)); newCase.setSteps(createSteps); if (method.isAnnotationPresent(Suite.class)) { long suiteId = method.getDeclaredAnnotation(Suite.class).id(); newCase.suiteId(suiteId); } Long id = Objects.requireNonNull(QASE_API.createCase(System.getProperty("QASE_PROJECT_CODE"), newCase).getResult()).getId(); log.info(String.format("New test case '%s' was created with @QaseId=%d", title, id)); } } else { log.warn("To create new test case in Qase.io please add @QaseTitle annotation"); } } else { log.warn("To create new test case in Qase.io please remove @QaseId annotation"); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/settings/listeners/QaseResultListener.java ================================================ package com.provectus.kafka.ui.settings.listeners; import static io.qase.api.utils.IntegrationUtils.getCaseId; import static io.qase.api.utils.IntegrationUtils.getCaseTitle; import static io.qase.api.utils.IntegrationUtils.getStacktrace; import static io.qase.client.model.ResultCreate.StatusEnum.FAILED; import static io.qase.client.model.ResultCreate.StatusEnum.PASSED; import static io.qase.client.model.ResultCreate.StatusEnum.SKIPPED; import io.qase.api.StepStorage; import io.qase.api.config.QaseConfig; import io.qase.api.services.QaseTestCaseListener; import io.qase.client.model.ResultCreate; import io.qase.client.model.ResultCreateCase; import io.qase.client.model.ResultCreateStepsInner; import io.qase.testng.guice.module.TestNgModule; import java.lang.reflect.Method; import java.util.LinkedList; import java.util.Optional; import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.testng.ITestContext; import org.testng.ITestListener; import org.testng.ITestResult; import org.testng.TestListenerAdapter; @Slf4j public class QaseResultListener extends TestListenerAdapter implements ITestListener { private static final String REPORTER_NAME = "TestNG"; static { System.setProperty(QaseConfig.QASE_CLIENT_REPORTER_NAME_KEY, REPORTER_NAME); } @Getter(lazy = true, value = AccessLevel.PRIVATE) private final QaseTestCaseListener qaseTestCaseListener = createQaseListener(); private static QaseTestCaseListener createQaseListener() { return TestNgModule.getInjector().getInstance(QaseTestCaseListener.class); } @Override public void onTestStart(ITestResult tr) { getQaseTestCaseListener().onTestCaseStarted(); super.onTestStart(tr); } @Override public void onTestSuccess(ITestResult tr) { getQaseTestCaseListener() .onTestCaseFinished(resultCreate -> setupResultItem(resultCreate, tr, PASSED)); super.onTestSuccess(tr); } @Override public void onTestSkipped(ITestResult tr) { getQaseTestCaseListener() .onTestCaseFinished(resultCreate -> setupResultItem(resultCreate, tr, SKIPPED)); super.onTestSuccess(tr); } @Override public void onTestFailure(ITestResult tr) { getQaseTestCaseListener() .onTestCaseFinished(resultCreate -> setupResultItem(resultCreate, tr, FAILED)); super.onTestFailure(tr); } @Override public void onFinish(ITestContext testContext) { getQaseTestCaseListener().onTestCasesSetFinished(); super.onFinish(testContext); } private void setupResultItem(ResultCreate resultCreate, ITestResult result, ResultCreate.StatusEnum status) { Optional resultThrowable = Optional.ofNullable(result.getThrowable()); String comment = resultThrowable .flatMap(throwable -> Optional.of(throwable.toString())).orElse(null); Boolean isDefect = resultThrowable .flatMap(throwable -> Optional.of(throwable instanceof AssertionError)) .orElse(false); String stacktrace = resultThrowable .flatMap(throwable -> Optional.of(getStacktrace(throwable))) .orElse(null); Method method = result.getMethod() .getConstructorOrMethod() .getMethod(); Long caseId = getCaseId(method); String caseTitle = null; if (caseId == null) { caseTitle = getCaseTitle(method); } LinkedList steps = StepStorage.stopSteps(); resultCreate ._case(caseTitle == null ? null : new ResultCreateCase().title(caseTitle)) .caseId(caseId) .status(status) .comment(comment) .stacktrace(stacktrace) .steps(steps.isEmpty() ? null : steps) .defect(isDefect); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/FileUtils.java ================================================ package com.provectus.kafka.ui.utilities; import static org.apache.kafka.common.utils.Utils.readFileAsString; import java.io.IOException; import java.nio.charset.StandardCharsets; import org.testcontainers.shaded.org.apache.commons.io.IOUtils; public class FileUtils { public static String getResourceAsString(String resourceFileName) { try { return IOUtils.resourceToString("/" + resourceFileName, StandardCharsets.UTF_8); } catch (IOException e) { throw new RuntimeException(e); } } public static String fileToString(String path) { try { return readFileAsString(path); } catch (IOException e) { throw new RuntimeException(e); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/StringUtils.java ================================================ package com.provectus.kafka.ui.utilities; import java.util.stream.IntStream; import lombok.extern.slf4j.Slf4j; @Slf4j public class StringUtils { public static String getMixedCase(String original) { return IntStream.range(0, original.length()) .mapToObj(i -> i % 2 == 0 ? Character.toUpperCase(original.charAt(i)) : original.charAt(i)) .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) .toString(); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/TimeUtils.java ================================================ package com.provectus.kafka.ui.utilities; import static com.codeborne.selenide.Selenide.sleep; import java.time.LocalTime; import lombok.extern.slf4j.Slf4j; @Slf4j public class TimeUtils { public static void waitUntilNewMinuteStarted() { int secondsLeft = 60 - LocalTime.now().getSecond(); log.debug("\nwaitUntilNewMinuteStarted: {}s", secondsLeft); sleep(secondsLeft * 1000); } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/WebUtils.java ================================================ package com.provectus.kafka.ui.utilities; import static com.codeborne.selenide.Selenide.executeJavaScript; import com.codeborne.selenide.Condition; import com.codeborne.selenide.SelenideElement; import com.codeborne.selenide.WebDriverRunner; import java.time.Duration; import lombok.extern.slf4j.Slf4j; import org.openqa.selenium.Keys; import org.openqa.selenium.interactions.Actions; @Slf4j public class WebUtils { public static int getTimeout(int... timeoutInSeconds) { return (timeoutInSeconds != null && timeoutInSeconds.length > 0) ? timeoutInSeconds[0] : 4; } public static void sendKeysAfterClear(SelenideElement element, String keys) { log.debug("\nsendKeysAfterClear: {} \nsend keys '{}'", element.getSearchCriteria(), keys); element.shouldBe(Condition.enabled).clear(); if (keys != null) { element.sendKeys(keys); } } public static void clickByActions(SelenideElement element) { log.debug("\nclickByActions: {}", element.getSearchCriteria()); element.shouldBe(Condition.enabled); new Actions(WebDriverRunner.getWebDriver()) .moveToElement(element) .click(element) .perform(); } public static void sendKeysByActions(SelenideElement element, String keys) { log.debug("\nsendKeysByActions: {} \nsend keys '{}'", element.getSearchCriteria(), keys); element.shouldBe(Condition.enabled); new Actions(WebDriverRunner.getWebDriver()) .moveToElement(element) .sendKeys(element, keys) .perform(); } public static void clickByJavaScript(SelenideElement element) { log.debug("\nclickByJavaScript: {}", element.getSearchCriteria()); element.shouldBe(Condition.enabled); String script = "arguments[0].click();"; executeJavaScript(script, element); } public static void clearByKeyboard(SelenideElement field) { log.debug("\nclearByKeyboard: {}", field.getSearchCriteria()); field.shouldBe(Condition.enabled).sendKeys(Keys.END); field.sendKeys(Keys.chord(Keys.CONTROL + "a"), Keys.DELETE); } public static boolean isVisible(SelenideElement element, int... timeoutInSeconds) { log.debug("\nisVisible: {}", element.getSearchCriteria()); boolean isVisible = false; try { element.shouldBe(Condition.visible, Duration.ofSeconds(getTimeout(timeoutInSeconds))); isVisible = true; } catch (Throwable e) { log.debug("{} is not visible", element.getSearchCriteria()); } return isVisible; } public static boolean isEnabled(SelenideElement element, int... timeoutInSeconds) { log.debug("\nisEnabled: {}", element.getSearchCriteria()); boolean isEnabled = false; try { element.shouldBe(Condition.enabled, Duration.ofSeconds(getTimeout(timeoutInSeconds))); isEnabled = true; } catch (Throwable e) { log.debug("{} is not enabled", element.getSearchCriteria()); } return isEnabled; } public static boolean isSelected(SelenideElement element, int... timeoutInSeconds) { log.debug("\nisSelected: {}", element.getSearchCriteria()); boolean isSelected = false; try { element.shouldBe(Condition.selected, Duration.ofSeconds(getTimeout(timeoutInSeconds))); isSelected = true; } catch (Throwable e) { log.debug("{} is not selected", element.getSearchCriteria()); } return isSelected; } public static void selectElement(SelenideElement element, boolean select) { if (select) { if (!element.isSelected()) { clickByJavaScript(element); } } else { if (element.isSelected()) { clickByJavaScript(element); } } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/QaseSetup.java ================================================ package com.provectus.kafka.ui.utilities.qase; import static com.provectus.kafka.ui.settings.BaseSource.SUITE_NAME; import static com.provectus.kafka.ui.variables.Suite.MANUAL; import static org.apache.commons.lang3.BooleanUtils.FALSE; import static org.apache.commons.lang3.BooleanUtils.TRUE; import static org.apache.commons.lang3.StringUtils.isEmpty; import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import lombok.extern.slf4j.Slf4j; @Slf4j public class QaseSetup { public static void qaseIntegrationSetup() { String qaseApiToken = System.getProperty("QASEIO_API_TOKEN"); if (isEmpty(qaseApiToken)) { log.warn("Integration with Qase is disabled due to run config or token wasn't defined."); System.setProperty("QASE_ENABLE", FALSE); } else { log.warn("Integration with Qase is enabled. Find this run at https://app.qase.io/run/KAFKAUI."); String automation = SUITE_NAME.equalsIgnoreCase(MANUAL) ? "" : "Automation "; System.setProperty("QASE_ENABLE", TRUE); System.setProperty("QASE_PROJECT_CODE", "KAFKAUI"); System.setProperty("QASE_API_TOKEN", qaseApiToken); System.setProperty("QASE_USE_BULK", TRUE); System.setProperty("QASE_RUN_NAME", DateTimeFormatter.ofPattern("dd.MM.yyyy HH:mm") .format(OffsetDateTime.now(ZoneOffset.UTC)) + ": " + automation + SUITE_NAME.toUpperCase() + " suite"); } } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/annotations/Automation.java ================================================ package com.provectus.kafka.ui.utilities.qase.annotations; import com.provectus.kafka.ui.utilities.qase.enums.State; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) public @interface Automation { State state(); } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/annotations/Status.java ================================================ package com.provectus.kafka.ui.utilities.qase.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) public @interface Status { com.provectus.kafka.ui.utilities.qase.enums.Status status(); } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/annotations/Suite.java ================================================ package com.provectus.kafka.ui.utilities.qase.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) public @interface Suite { long id(); } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/enums/State.java ================================================ package com.provectus.kafka.ui.utilities.qase.enums; public enum State { NOT_AUTOMATED(0), TO_BE_AUTOMATED(1), AUTOMATED(2); private final int value; State(int value) { this.value = value; } public int getValue() { return value; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/utilities/qase/enums/Status.java ================================================ package com.provectus.kafka.ui.utilities.qase.enums; public enum Status { ACTUAL(0), DRAFT(1), DEPRECATED(2); private final int value; Status(int value) { this.value = value; } public int getValue() { return value; } } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/variables/Browser.java ================================================ package com.provectus.kafka.ui.variables; public interface Browser { String CONTAINER = "container"; String LOCAL = "local"; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/variables/Expected.java ================================================ package com.provectus.kafka.ui.variables; public interface Expected { String BROKER_SOURCE_INFO_TOOLTIP = "DYNAMIC_TOPIC_CONFIG = dynamic topic config that is configured for a specific topic\n" + "DYNAMIC_BROKER_LOGGER_CONFIG = dynamic broker logger config that is configured for a specific broker\n" + "DYNAMIC_BROKER_CONFIG = dynamic broker config that is configured for a specific broker\n" + "DYNAMIC_DEFAULT_BROKER_CONFIG = dynamic broker config that is configured as default " + "for all brokers in the cluster\n" + "STATIC_BROKER_CONFIG = static broker config provided as broker properties at start up " + "(e.g. server.properties file)\n" + "DEFAULT_CONFIG = built-in default configuration for configs that have a default value\n" + "UNKNOWN = source unknown e.g. in the ConfigEntry used for alter requests where source is not set"; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/variables/Suite.java ================================================ package com.provectus.kafka.ui.variables; public interface Suite { String CUSTOM = "custom"; String MANUAL = "manual"; String REGRESSION = "regression"; String SANITY = "sanity"; String SMOKE = "smoke"; } ================================================ FILE: kafka-ui-e2e-checks/src/main/java/com/provectus/kafka/ui/variables/Url.java ================================================ package com.provectus.kafka.ui.variables; public interface Url { String BROKERS_LIST_URL = "http://%s:8080/ui/clusters/local/brokers"; String TOPICS_LIST_URL = "http://%s:8080/ui/clusters/local/all-topics"; String CONSUMERS_LIST_URL = "http://%s:8080/ui/clusters/local/consumer-groups"; String SCHEMA_REGISTRY_LIST_URL = "http://%s:8080/ui/clusters/local/schemas"; String KAFKA_CONNECT_LIST_URL = "http://%s:8080/ui/clusters/local/connectors"; String KSQL_DB_LIST_URL = "http://%s:8080/ui/clusters/local/ksqldb/tables"; } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/allure.properties ================================================ allure.results.directory=allure-results allure.link.issue.pattern=https://github.com/provectus/kafka-ui/issues/{} ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/connectors/config_for_create_connector.json ================================================ { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "topic_for_connector", "table.name.format": "sink_activities_e2e_test_connector_creating", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085", "auto.create": "true", "pk.mode": "record_value", "pk.fields": "id", "insert.mode": "upsert", "errors.log.enable": "true", "errors.log.include.messages": "true" } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/connectors/config_for_create_connector_via_api.json ================================================ { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "topic_for_connector" } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/connectors/config_for_update_connector.json ================================================ { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "topic_for_update_connector", "table.name.format": "sink_activities_e2e_test_connector_updating", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085", "auto.create": "true", "pk.mode": "record_value", "pk.fields": "id", "insert.mode": "upsert", "errors.log.enable": "true", "errors.log.include.messages": "true" } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/connectors/delete_connector_config.json ================================================ { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "topic_for_delete_connector", "table.name.format": "sink_activities_e2e_test_connector_deleting", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "key.converter.schema.registry.url": "http://schemaregistry0:8085", "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schema.registry.url": "http://schemaregistry0:8085", "auto.create": "false", "pk.mode": "record_value", "pk.fields": "id", "insert.mode": "upsert", "errors.log.enable": "true", "errors.log.include.messages": "true" } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/schemas/schema_avro_for_update.json ================================================ { "type": "record", "name": "Message", "namespace": "com.provectus.kafka", "fields": [ { "name": "text", "type": "string", "default": null }, { "name": "value", "type": "string", "default": null } ] } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/schemas/schema_avro_value.json ================================================ { "type": "record", "name": "Student", "namespace": "DataFlair", "fields": [ { "name": "Name", "type": "string" }, { "name": "Age", "type": "int" } ] } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/schemas/schema_json_Value.json ================================================ { "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector", "connection.url": "jdbc:postgresql://postgres-db:5432/test", "connection.user": "dev_user", "connection.password": "12345", "topics": "topic_for_connector" } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/schemas/schema_protobuf_value.txt ================================================ enum SchemaType { AVRO = 0; JSON = 1; PROTOBUF = 2; } ================================================ FILE: kafka-ui-e2e-checks/src/main/resources/testData/topics/message_content_create_topic.json ================================================ { "schema": { "type":"struct", "fields": [ { "type":"string", "optional":false, "field":"id" },{ "type":"string", "optional":false, "field":"value" } ], "optional":false, "name":"test" }, "payload": { "id":"1", "value":"kafka" } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/BaseTest.java ================================================ package com.provectus.kafka.ui; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.BROKERS; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.CONSUMERS; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KAFKA_CONNECT; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KSQL_DB; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.SCHEMA_REGISTRY; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.TOPICS; import static com.provectus.kafka.ui.settings.BaseSource.BASE_UI_URL; import static com.provectus.kafka.ui.settings.drivers.WebDriver.browserClear; import static com.provectus.kafka.ui.settings.drivers.WebDriver.browserQuit; import static com.provectus.kafka.ui.settings.drivers.WebDriver.browserSetup; import static com.provectus.kafka.ui.settings.drivers.WebDriver.loggerSetup; import static com.provectus.kafka.ui.utilities.qase.QaseSetup.qaseIntegrationSetup; import com.codeborne.selenide.Condition; import com.codeborne.selenide.Selenide; import com.codeborne.selenide.SelenideElement; import com.provectus.kafka.ui.settings.listeners.AllureListener; import com.provectus.kafka.ui.settings.listeners.LoggerListener; import com.provectus.kafka.ui.settings.listeners.QaseResultListener; import io.qameta.allure.Step; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Listeners; import org.testng.asserts.SoftAssert; @Slf4j @Listeners({AllureListener.class, LoggerListener.class, QaseResultListener.class}) public abstract class BaseTest extends Facade { @BeforeSuite(alwaysRun = true) public void beforeSuite() { qaseIntegrationSetup(); loggerSetup(); browserSetup(); } @AfterSuite(alwaysRun = true) public void afterSuite() { browserQuit(); } @BeforeMethod(alwaysRun = true) public void beforeMethod() { Selenide.open(BASE_UI_URL); naviSideBar.waitUntilScreenReady(); } @AfterMethod(alwaysRun = true) public void afterMethod() { browserClear(); } @Step protected void navigateToBrokers() { naviSideBar .openSideMenu(BROKERS); brokersList .waitUntilScreenReady(); } @Step protected void navigateToBrokersAndOpenDetails(int brokerId) { naviSideBar .openSideMenu(BROKERS); brokersList .waitUntilScreenReady() .openBroker(brokerId); brokersDetails .waitUntilScreenReady(); } @Step protected void navigateToTopics() { naviSideBar .openSideMenu(TOPICS); topicsList .waitUntilScreenReady() .setShowInternalRadioButton(false); } @Step protected void navigateToTopicsAndOpenDetails(String topicName) { navigateToTopics(); topicsList .openTopic(topicName); topicDetails .waitUntilScreenReady(); } @Step protected void navigateToConsumers() { naviSideBar .openSideMenu(CONSUMERS); consumersList .waitUntilScreenReady(); } @Step protected void navigateToSchemaRegistry() { naviSideBar .openSideMenu(SCHEMA_REGISTRY); schemaRegistryList .waitUntilScreenReady(); } @Step protected void navigateToSchemaRegistryAndOpenDetails(String schemaName) { navigateToSchemaRegistry(); schemaRegistryList .openSchema(schemaName); schemaDetails .waitUntilScreenReady(); } @Step protected void navigateToConnectors() { naviSideBar .openSideMenu(KAFKA_CONNECT); kafkaConnectList .waitUntilScreenReady(); } @Step protected void navigateToConnectorsAndOpenDetails(String connectorName) { navigateToConnectors(); kafkaConnectList .openConnector(connectorName); connectorDetails .waitUntilScreenReady(); } @Step protected void navigateToKsqlDb() { naviSideBar .openSideMenu(KSQL_DB); ksqlDbList .waitUntilScreenReady(); } @Step protected void verifyElementsCondition(List elementList, Condition expectedCondition) { SoftAssert softly = new SoftAssert(); elementList.forEach(element -> softly.assertTrue(element.is(expectedCondition), element.getSearchCriteria() + " is " + expectedCondition)); softly.assertAll(); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/Facade.java ================================================ package com.provectus.kafka.ui; import com.provectus.kafka.ui.pages.brokers.BrokersConfigTab; import com.provectus.kafka.ui.pages.brokers.BrokersDetails; import com.provectus.kafka.ui.pages.brokers.BrokersList; import com.provectus.kafka.ui.pages.connectors.ConnectorCreateForm; import com.provectus.kafka.ui.pages.connectors.ConnectorDetails; import com.provectus.kafka.ui.pages.connectors.KafkaConnectList; import com.provectus.kafka.ui.pages.consumers.ConsumersDetails; import com.provectus.kafka.ui.pages.consumers.ConsumersList; import com.provectus.kafka.ui.pages.ksqldb.KsqlDbList; import com.provectus.kafka.ui.pages.ksqldb.KsqlQueryForm; import com.provectus.kafka.ui.pages.panels.NaviSideBar; import com.provectus.kafka.ui.pages.panels.TopPanel; import com.provectus.kafka.ui.pages.schemas.SchemaCreateForm; import com.provectus.kafka.ui.pages.schemas.SchemaDetails; import com.provectus.kafka.ui.pages.schemas.SchemaRegistryList; import com.provectus.kafka.ui.pages.topics.ProduceMessagePanel; import com.provectus.kafka.ui.pages.topics.TopicCreateEditForm; import com.provectus.kafka.ui.pages.topics.TopicDetails; import com.provectus.kafka.ui.pages.topics.TopicSettingsTab; import com.provectus.kafka.ui.pages.topics.TopicsList; import com.provectus.kafka.ui.services.ApiService; public abstract class Facade { protected ApiService apiService = new ApiService(); protected ConnectorCreateForm connectorCreateForm = new ConnectorCreateForm(); protected KafkaConnectList kafkaConnectList = new KafkaConnectList(); protected ConnectorDetails connectorDetails = new ConnectorDetails(); protected SchemaCreateForm schemaCreateForm = new SchemaCreateForm(); protected SchemaDetails schemaDetails = new SchemaDetails(); protected SchemaRegistryList schemaRegistryList = new SchemaRegistryList(); protected ProduceMessagePanel produceMessagePanel = new ProduceMessagePanel(); protected TopicCreateEditForm topicCreateEditForm = new TopicCreateEditForm(); protected TopicsList topicsList = new TopicsList(); protected TopicDetails topicDetails = new TopicDetails(); protected ConsumersDetails consumersDetails = new ConsumersDetails(); protected ConsumersList consumersList = new ConsumersList(); protected NaviSideBar naviSideBar = new NaviSideBar(); protected TopPanel topPanel = new TopPanel(); protected BrokersList brokersList = new BrokersList(); protected BrokersDetails brokersDetails = new BrokersDetails(); protected BrokersConfigTab brokersConfigTab = new BrokersConfigTab(); protected TopicSettingsTab topicSettingsTab = new TopicSettingsTab(); protected KsqlQueryForm ksqlQueryForm = new KsqlQueryForm(); protected KsqlDbList ksqlDbList = new KsqlDbList(); } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/BaseManualTest.java ================================================ package com.provectus.kafka.ui.manualsuite; import static com.provectus.kafka.ui.utilities.qase.QaseSetup.qaseIntegrationSetup; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import static com.provectus.kafka.ui.utilities.qase.enums.State.TO_BE_AUTOMATED; import com.provectus.kafka.ui.settings.listeners.QaseResultListener; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import java.lang.reflect.Method; import org.testng.SkipException; import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Listeners; @Listeners(QaseResultListener.class) public abstract class BaseManualTest { @BeforeSuite public void beforeSuite() { qaseIntegrationSetup(); } @BeforeMethod public void beforeMethod(Method method) { if (method.getAnnotation(Automation.class).state().equals(NOT_AUTOMATED) || method.getAnnotation(Automation.class).state().equals(TO_BE_AUTOMATED)) { throw new SkipException("Skip test exception"); } } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SanityBacklog.java ================================================ package com.provectus.kafka.ui.manualsuite.backlog; import com.provectus.kafka.ui.manualsuite.BaseManualTest; public class SanityBacklog extends BaseManualTest { } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/backlog/SmokeBacklog.java ================================================ package com.provectus.kafka.ui.manualsuite.backlog; import static com.provectus.kafka.ui.qasesuite.BaseQaseTest.SCHEMAS_SUITE_ID; import static com.provectus.kafka.ui.qasesuite.BaseQaseTest.TOPICS_PROFILE_SUITE_ID; import static com.provectus.kafka.ui.qasesuite.BaseQaseTest.TOPICS_SUITE_ID; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import static com.provectus.kafka.ui.utilities.qase.enums.State.TO_BE_AUTOMATED; import com.provectus.kafka.ui.manualsuite.BaseManualTest; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import com.provectus.kafka.ui.utilities.qase.annotations.Suite; import io.qase.api.annotation.QaseId; import org.testng.annotations.Test; public class SmokeBacklog extends BaseManualTest { @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(335) @Test public void testCaseA() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(336) @Test public void testCaseB() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(343) @Test public void testCaseC() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = SCHEMAS_SUITE_ID) @QaseId(345) @Test public void testCaseD() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = SCHEMAS_SUITE_ID) @QaseId(346) @Test public void testCaseE() { } @Automation(state = TO_BE_AUTOMATED) @Suite(id = TOPICS_PROFILE_SUITE_ID) @QaseId(347) @Test public void testCaseF() { } @Automation(state = NOT_AUTOMATED) @Suite(id = TOPICS_SUITE_ID) @QaseId(50) @Test public void testCaseG() { } @Automation(state = NOT_AUTOMATED) @Suite(id = SCHEMAS_SUITE_ID) @QaseId(351) @Test public void testCaseH() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/suite/DataMaskingTest.java ================================================ package com.provectus.kafka.ui.manualsuite.suite; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import com.provectus.kafka.ui.manualsuite.BaseManualTest; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import io.qase.api.annotation.QaseId; import org.testng.annotations.Test; public class DataMaskingTest extends BaseManualTest { @Automation(state = NOT_AUTOMATED) @QaseId(262) @Test public void testCaseA() { } @Automation(state = NOT_AUTOMATED) @QaseId(264) @Test public void testCaseB() { } @Automation(state = NOT_AUTOMATED) @QaseId(265) @Test public void testCaseC() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/suite/RbacTest.java ================================================ package com.provectus.kafka.ui.manualsuite.suite; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import com.provectus.kafka.ui.manualsuite.BaseManualTest; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import io.qase.api.annotation.QaseId; import org.testng.annotations.Test; public class RbacTest extends BaseManualTest { @Automation(state = NOT_AUTOMATED) @QaseId(249) @Test public void testCaseA() { } @Automation(state = NOT_AUTOMATED) @QaseId(251) @Test public void testCaseB() { } @Automation(state = NOT_AUTOMATED) @QaseId(257) @Test public void testCaseC() { } @Automation(state = NOT_AUTOMATED) @QaseId(258) @Test public void testCaseD() { } @Automation(state = NOT_AUTOMATED) @QaseId(259) @Test public void testCaseE() { } @Automation(state = NOT_AUTOMATED) @QaseId(260) @Test public void testCaseF() { } @Automation(state = NOT_AUTOMATED) @QaseId(261) @Test public void testCaseG() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/suite/TopicsTest.java ================================================ package com.provectus.kafka.ui.manualsuite.suite; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import com.provectus.kafka.ui.manualsuite.BaseManualTest; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import io.qase.api.annotation.QaseId; import org.testng.annotations.Test; public class TopicsTest extends BaseManualTest { @Automation(state = NOT_AUTOMATED) @QaseId(17) @Test public void testCaseA() { } @Automation(state = NOT_AUTOMATED) @QaseId(18) @Test public void testCaseB() { } @Automation(state = NOT_AUTOMATED) @QaseId(21) @Test() public void testCaseC() { } @Automation(state = NOT_AUTOMATED) @QaseId(22) @Test public void testCaseD() { } @Automation(state = NOT_AUTOMATED) @QaseId(47) @Test public void testCaseE() { } @Automation(state = NOT_AUTOMATED) @QaseId(48) @Test public void testCaseF() { } @Automation(state = NOT_AUTOMATED) @QaseId(49) @Test public void testCaseG() { } @Automation(state = NOT_AUTOMATED) @QaseId(57) @Test public void testCaseH() { } @Automation(state = NOT_AUTOMATED) @QaseId(58) @Test public void testCaseI() { } @Automation(state = NOT_AUTOMATED) @QaseId(269) @Test public void testCaseJ() { } @Automation(state = NOT_AUTOMATED) @QaseId(270) @Test public void testCaseK() { } @Automation(state = NOT_AUTOMATED) @QaseId(271) @Test public void testCaseL() { } @Automation(state = NOT_AUTOMATED) @QaseId(272) @Test public void testCaseM() { } @Automation(state = NOT_AUTOMATED) @QaseId(337) @Test public void testCaseN() { } @Automation(state = NOT_AUTOMATED) @QaseId(339) @Test public void testCaseO() { } @Automation(state = NOT_AUTOMATED) @QaseId(341) @Test public void testCaseP() { } @Automation(state = NOT_AUTOMATED) @QaseId(342) @Test public void testCaseQ() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/manualsuite/suite/WizardTest.java ================================================ package com.provectus.kafka.ui.manualsuite.suite; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import com.provectus.kafka.ui.manualsuite.BaseManualTest; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import io.qase.api.annotation.QaseId; import org.testng.annotations.Test; public class WizardTest extends BaseManualTest { @Automation(state = NOT_AUTOMATED) @QaseId(333) @Test public void testCaseA() { } @Automation(state = NOT_AUTOMATED) @QaseId(338) @Test public void testCaseB() { } @Automation(state = NOT_AUTOMATED) @QaseId(340) @Test public void testCaseC() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qasesuite/BaseQaseTest.java ================================================ package com.provectus.kafka.ui.qasesuite; import static com.provectus.kafka.ui.utilities.qase.QaseSetup.qaseIntegrationSetup; import com.provectus.kafka.ui.settings.listeners.QaseCreateListener; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Listeners; @Listeners(QaseCreateListener.class) public abstract class BaseQaseTest { public static final long BROKERS_SUITE_ID = 1; public static final long CONNECTORS_SUITE_ID = 10; public static final long KSQL_DB_SUITE_ID = 8; public static final long SANITY_SUITE_ID = 19; public static final long SCHEMAS_SUITE_ID = 11; public static final long TOPICS_SUITE_ID = 2; public static final long TOPICS_CREATE_SUITE_ID = 4; public static final long TOPICS_PROFILE_SUITE_ID = 5; @BeforeSuite public void beforeSuite() { qaseIntegrationSetup(); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qasesuite/Template.java ================================================ package com.provectus.kafka.ui.qasesuite; import static com.provectus.kafka.ui.utilities.qase.enums.State.NOT_AUTOMATED; import static com.provectus.kafka.ui.utilities.qase.enums.Status.DRAFT; import com.provectus.kafka.ui.utilities.qase.annotations.Automation; import com.provectus.kafka.ui.utilities.qase.annotations.Status; import com.provectus.kafka.ui.utilities.qase.annotations.Suite; import io.qase.api.annotation.QaseTitle; import io.qase.api.annotation.Step; public class Template extends BaseQaseTest { /** * this class is a kind of placeholder or example, use is as template to create new one * copy Template into kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/qaseSuite/ * place it into regarding folder and rename according to test case summary from Qase.io * uncomment @Test and set all annotations according to kafka-ui-e2e-checks/QASE.md */ @Automation(state = NOT_AUTOMATED) @QaseTitle("testCaseA title") @Status(status = DRAFT) @Suite(id = 0) // @org.testng.annotations.Test public void testCaseA() { stepA(); stepB(); stepC(); stepD(); stepE(); stepF(); } @Step("stepA action") private void stepA() { } @Step("stepB action") private void stepB() { } @Step("stepC action") private void stepC() { } @Step("stepD action") private void stepD() { } @Step("stepE action") private void stepE() { } @Step("stepF action") private void stepF() { } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/sanitysuite/TopicsTest.java ================================================ package com.provectus.kafka.ui.sanitysuite; import static com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue.COMPACT; import static com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue.DELETE; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.models.Topic; import io.qase.api.annotation.QaseId; import java.util.ArrayList; import java.util.List; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; public class TopicsTest extends BaseTest { private static final List TOPIC_LIST = new ArrayList<>(); @QaseId(285) @Test() public void verifyClearMessagesMenuStateAfterTopicUpdate() { Topic topic = new Topic() .setName("topic-" + randomAlphabetic(5)) .setNumberOfPartitions(1) .setCleanupPolicyValue(DELETE); navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(topic.getName()) .setNumberOfPartitions(topic.getNumberOfPartitions()) .selectCleanupPolicy(topic.getCleanupPolicyValue()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady(); TOPIC_LIST.add(topic); topicDetails .openDotMenu(); Assert.assertTrue(topicDetails.isClearMessagesMenuEnabled(), "isClearMessagesMenuEnabled"); topic.setCleanupPolicyValue(COMPACT); editCleanUpPolicyAndOpenDotMenu(topic); Assert.assertFalse(topicDetails.isClearMessagesMenuEnabled(), "isClearMessagesMenuEnabled"); topic.setCleanupPolicyValue(DELETE); editCleanUpPolicyAndOpenDotMenu(topic); Assert.assertTrue(topicDetails.isClearMessagesMenuEnabled(), "isClearMessagesMenuEnabled"); } private void editCleanUpPolicyAndOpenDotMenu(Topic topic) { topicDetails .clickEditSettingsMenu(); topicCreateEditForm .waitUntilScreenReady() .selectCleanupPolicy(topic.getCleanupPolicyValue()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady() .openDotMenu(); } @AfterClass(alwaysRun = true) public void afterClass() { TOPIC_LIST.forEach(topic -> apiService.deleteTopic(topic.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/SmokeTest.java ================================================ package com.provectus.kafka.ui.smokesuite; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.BROKERS; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.KAFKA_CONNECT; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.SCHEMA_REGISTRY; import static com.provectus.kafka.ui.pages.panels.enums.MenuItem.TOPICS; import static com.provectus.kafka.ui.settings.BaseSource.BASE_HOST; import static com.provectus.kafka.ui.utilities.FileUtils.getResourceAsString; import static com.provectus.kafka.ui.variables.Url.BROKERS_LIST_URL; import static com.provectus.kafka.ui.variables.Url.CONSUMERS_LIST_URL; import static com.provectus.kafka.ui.variables.Url.KAFKA_CONNECT_LIST_URL; import static com.provectus.kafka.ui.variables.Url.KSQL_DB_LIST_URL; import static com.provectus.kafka.ui.variables.Url.SCHEMA_REGISTRY_LIST_URL; import static com.provectus.kafka.ui.variables.Url.TOPICS_LIST_URL; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.codeborne.selenide.Condition; import com.codeborne.selenide.WebDriverRunner; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.models.Connector; import com.provectus.kafka.ui.models.Schema; import com.provectus.kafka.ui.models.Topic; import com.provectus.kafka.ui.pages.panels.enums.MenuItem; import io.qameta.allure.Step; import io.qase.api.annotation.QaseId; import java.util.stream.Collectors; import java.util.stream.Stream; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class SmokeTest extends BaseTest { private static final int BROKER_ID = 1; private static final Schema TEST_SCHEMA = Schema.createSchemaAvro(); private static final Topic TEST_TOPIC = new Topic() .setName("new-topic-" + randomAlphabetic(5)) .setNumberOfPartitions(1); private static final Connector TEST_CONNECTOR = new Connector() .setName("new-connector-" + randomAlphabetic(5)) .setConfig(getResourceAsString("testData/connectors/config_for_create_connector_via_api.json")); @BeforeClass(alwaysRun = true) public void beforeClass() { apiService .createTopic(TEST_TOPIC) .createSchema(TEST_SCHEMA) .createConnector(TEST_CONNECTOR); } @QaseId(198) @Test public void checkBasePageElements() { verifyElementsCondition( Stream.concat(topPanel.getAllVisibleElements().stream(), naviSideBar.getAllMenuButtons().stream()) .collect(Collectors.toList()), Condition.visible); verifyElementsCondition( Stream.concat(topPanel.getAllEnabledElements().stream(), naviSideBar.getAllMenuButtons().stream()) .collect(Collectors.toList()), Condition.enabled); } @QaseId(45) @Test public void checkUrlWhileNavigating() { navigateToBrokers(); verifyCurrentUrl(BROKERS_LIST_URL); navigateToTopics(); verifyCurrentUrl(TOPICS_LIST_URL); navigateToConsumers(); verifyCurrentUrl(CONSUMERS_LIST_URL); navigateToSchemaRegistry(); verifyCurrentUrl(SCHEMA_REGISTRY_LIST_URL); navigateToConnectors(); verifyCurrentUrl(KAFKA_CONNECT_LIST_URL); navigateToKsqlDb(); verifyCurrentUrl(KSQL_DB_LIST_URL); } @QaseId(46) @Test public void checkPathWhileNavigating() { navigateToBrokersAndOpenDetails(BROKER_ID); verifyComponentsPath(BROKERS, String.format("Broker %d", BROKER_ID)); navigateToTopicsAndOpenDetails(TEST_TOPIC.getName()); verifyComponentsPath(TOPICS, TEST_TOPIC.getName()); navigateToSchemaRegistryAndOpenDetails(TEST_SCHEMA.getName()); verifyComponentsPath(SCHEMA_REGISTRY, TEST_SCHEMA.getName()); navigateToConnectorsAndOpenDetails(TEST_CONNECTOR.getName()); verifyComponentsPath(KAFKA_CONNECT, TEST_CONNECTOR.getName()); } @Step private void verifyCurrentUrl(String expectedUrl) { String urlWithoutParameters = WebDriverRunner.getWebDriver().getCurrentUrl(); if (urlWithoutParameters.contains("?")) { urlWithoutParameters = urlWithoutParameters.substring(0, urlWithoutParameters.indexOf("?")); } Assert.assertEquals(urlWithoutParameters, String.format(expectedUrl, BASE_HOST), "getCurrentUrl()"); } @Step private void verifyComponentsPath(MenuItem menuItem, String expectedPath) { Assert.assertEquals(naviSideBar.getPagePath(menuItem), expectedPath, String.format("getPagePath() for %s", menuItem.getPageTitle().toUpperCase())); } @AfterClass(alwaysRun = true) public void afterClass() { apiService .deleteTopic(TEST_TOPIC.getName()) .deleteSchema(TEST_SCHEMA.getName()) .deleteConnector(TEST_CONNECTOR.getName()); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/brokers/BrokersTest.java ================================================ package com.provectus.kafka.ui.smokesuite.brokers; import static com.provectus.kafka.ui.pages.brokers.BrokersDetails.DetailsTab.CONFIGS; import static com.provectus.kafka.ui.utilities.StringUtils.getMixedCase; import static com.provectus.kafka.ui.variables.Expected.BROKER_SOURCE_INFO_TOOLTIP; import com.codeborne.selenide.Condition; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.pages.brokers.BrokersConfigTab; import io.qameta.allure.Issue; import io.qase.api.annotation.QaseId; import java.util.List; import org.testng.Assert; import org.testng.annotations.Ignore; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; public class BrokersTest extends BaseTest { public static final int DEFAULT_BROKER_ID = 1; @QaseId(1) @Test public void checkBrokersOverview() { navigateToBrokers(); Assert.assertTrue(brokersList.getAllBrokers().size() > 0, "getAllBrokers()"); verifyElementsCondition(brokersList.getAllVisibleElements(), Condition.visible); verifyElementsCondition(brokersList.getAllEnabledElements(), Condition.enabled); } @QaseId(85) @Test public void checkExistingBrokersInCluster() { navigateToBrokers(); Assert.assertTrue(brokersList.getAllBrokers().size() > 0, "getAllBrokers()"); brokersList .openBroker(DEFAULT_BROKER_ID); brokersDetails .waitUntilScreenReady(); verifyElementsCondition(brokersDetails.getAllVisibleElements(), Condition.visible); verifyElementsCondition(brokersDetails.getAllEnabledElements(), Condition.enabled); brokersDetails .openDetailsTab(CONFIGS); brokersConfigTab .waitUntilScreenReady(); verifyElementsCondition(brokersConfigTab.getColumnHeaders(), Condition.visible); verifyElementsCondition(brokersConfigTab.getEditButtons(), Condition.enabled); Assert.assertTrue(brokersConfigTab.isSearchByKeyVisible(), "isSearchByKeyVisible()"); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3347") @QaseId(330) @Test public void brokersConfigFirstPageSearchCheck() { navigateToBrokersAndOpenDetails(DEFAULT_BROKER_ID); brokersDetails .openDetailsTab(CONFIGS); String anyConfigKeyFirstPage = brokersConfigTab .getAllConfigs().stream() .findAny().orElseThrow() .getKey(); brokersConfigTab .clickNextButton(); Assert.assertFalse(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeyFirstPage), String.format("getAllConfigs().contains(%s)", anyConfigKeyFirstPage)); brokersConfigTab .searchConfig(anyConfigKeyFirstPage); Assert.assertTrue(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeyFirstPage), String.format("getAllConfigs().contains(%s)", anyConfigKeyFirstPage)); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3347") @QaseId(350) @Test public void brokersConfigSecondPageSearchCheck() { navigateToBrokersAndOpenDetails(DEFAULT_BROKER_ID); brokersDetails .openDetailsTab(CONFIGS); brokersConfigTab .clickNextButton(); String anyConfigKeySecondPage = brokersConfigTab .getAllConfigs().stream() .findAny().orElseThrow() .getKey(); brokersConfigTab .clickPreviousButton(); Assert.assertFalse(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeySecondPage), String.format("getAllConfigs().contains(%s)", anyConfigKeySecondPage)); brokersConfigTab .searchConfig(anyConfigKeySecondPage); Assert.assertTrue(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeySecondPage), String.format("getAllConfigs().contains(%s)", anyConfigKeySecondPage)); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3347") @QaseId(348) @Test public void brokersConfigCaseInsensitiveSearchCheck() { navigateToBrokersAndOpenDetails(DEFAULT_BROKER_ID); brokersDetails .openDetailsTab(CONFIGS); String anyConfigKeyFirstPage = brokersConfigTab .getAllConfigs().stream() .findAny().orElseThrow() .getKey(); brokersConfigTab .clickNextButton(); Assert.assertFalse(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeyFirstPage), String.format("getAllConfigs().contains(%s)", anyConfigKeyFirstPage)); SoftAssert softly = new SoftAssert(); List.of(anyConfigKeyFirstPage.toLowerCase(), anyConfigKeyFirstPage.toUpperCase(), getMixedCase(anyConfigKeyFirstPage)) .forEach(configCase -> { brokersConfigTab .searchConfig(configCase); softly.assertTrue(brokersConfigTab.getAllConfigs().stream() .map(BrokersConfigTab.BrokersConfigItem::getKey) .toList().contains(anyConfigKeyFirstPage), String.format("getAllConfigs().contains(%s)", configCase)); }); softly.assertAll(); } @QaseId(331) @Test public void brokersSourceInfoCheck() { navigateToBrokersAndOpenDetails(DEFAULT_BROKER_ID); brokersDetails .openDetailsTab(CONFIGS); String sourceInfoTooltip = brokersConfigTab .hoverOnSourceInfoIcon() .getSourceInfoTooltipText(); Assert.assertEquals(sourceInfoTooltip, BROKER_SOURCE_INFO_TOOLTIP, "brokerSourceInfoTooltip"); } @QaseId(332) @Test public void brokersConfigEditCheck() { navigateToBrokersAndOpenDetails(DEFAULT_BROKER_ID); brokersDetails .openDetailsTab(CONFIGS); String configKey = "log.cleaner.min.compaction.lag.ms"; BrokersConfigTab.BrokersConfigItem configItem = brokersConfigTab .searchConfig(configKey) .getConfig(configKey); int defaultValue = Integer.parseInt(configItem.getValue()); configItem .clickEditBtn(); SoftAssert softly = new SoftAssert(); softly.assertTrue(configItem.getSaveBtn().isDisplayed(), "getSaveBtn().isDisplayed()"); softly.assertTrue(configItem.getCancelBtn().isDisplayed(), "getCancelBtn().isDisplayed()"); softly.assertTrue(configItem.getValueFld().isEnabled(), "getValueFld().isEnabled()"); softly.assertAll(); int newValue = defaultValue + 1; configItem .setValue(String.valueOf(newValue)) .clickCancelBtn(); Assert.assertEquals(Integer.parseInt(configItem.getValue()), defaultValue, "getValue()"); configItem .clickEditBtn() .setValue(String.valueOf(newValue)) .clickSaveBtn() .clickConfirm(); configItem = brokersConfigTab .searchConfig(configKey) .getConfig(configKey); softly.assertFalse(configItem.getSaveBtn().isDisplayed(), "getSaveBtn().isDisplayed()"); softly.assertFalse(configItem.getCancelBtn().isDisplayed(), "getCancelBtn().isDisplayed()"); softly.assertTrue(configItem.getEditBtn().isDisplayed(), "getEditBtn().isDisplayed()"); softly.assertEquals(Integer.parseInt(configItem.getValue()), newValue, "getValue()"); softly.assertAll(); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/connectors/ConnectorsTest.java ================================================ package com.provectus.kafka.ui.smokesuite.connectors; import static com.provectus.kafka.ui.pages.BasePage.AlertHeader.SUCCESS; import static com.provectus.kafka.ui.utilities.FileUtils.getResourceAsString; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.models.Connector; import com.provectus.kafka.ui.models.Topic; import io.qase.api.annotation.QaseId; import java.util.ArrayList; import java.util.List; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; public class ConnectorsTest extends BaseTest { private static final List TOPIC_LIST = new ArrayList<>(); private static final List CONNECTOR_LIST = new ArrayList<>(); private static final String MESSAGE_CONTENT = "testData/topics/message_content_create_topic.json"; private static final String MESSAGE_KEY = " "; private static final Topic TOPIC_FOR_CREATE = new Topic() .setName("topic-for-create-connector-" + randomAlphabetic(5)) .setMessageValue(MESSAGE_CONTENT).setMessageKey(MESSAGE_KEY); private static final Topic TOPIC_FOR_DELETE = new Topic() .setName("topic-for-delete-connector-" + randomAlphabetic(5)) .setMessageValue(MESSAGE_CONTENT).setMessageKey(MESSAGE_KEY); private static final Topic TOPIC_FOR_UPDATE = new Topic() .setName("topic-for-update-connector-" + randomAlphabetic(5)) .setMessageValue(MESSAGE_CONTENT).setMessageKey(MESSAGE_KEY); private static final Connector CONNECTOR_FOR_DELETE = new Connector() .setName("connector-for-delete-" + randomAlphabetic(5)) .setConfig(getResourceAsString("testData/connectors/delete_connector_config.json")); private static final Connector CONNECTOR_FOR_UPDATE = new Connector() .setName("connector-for-update-and-delete-" + randomAlphabetic(5)) .setConfig(getResourceAsString("testData/connectors/config_for_create_connector_via_api.json")); @BeforeClass(alwaysRun = true) public void beforeClass() { TOPIC_LIST.addAll(List.of(TOPIC_FOR_CREATE, TOPIC_FOR_DELETE, TOPIC_FOR_UPDATE)); TOPIC_LIST.forEach(topic -> apiService .createTopic(topic) .sendMessage(topic) ); CONNECTOR_LIST.addAll(List.of(CONNECTOR_FOR_DELETE, CONNECTOR_FOR_UPDATE)); CONNECTOR_LIST.forEach(connector -> apiService.createConnector(connector)); } @QaseId(42) @Test public void createConnector() { Connector connectorForCreate = new Connector() .setName("connector-for-create-" + randomAlphabetic(5)) .setConfig(getResourceAsString("testData/connectors/config_for_create_connector.json")); navigateToConnectors(); kafkaConnectList .clickCreateConnectorBtn(); connectorCreateForm .waitUntilScreenReady() .setConnectorDetails(connectorForCreate.getName(), connectorForCreate.getConfig()) .clickSubmitButton(); connectorDetails .waitUntilScreenReady(); navigateToConnectorsAndOpenDetails(connectorForCreate.getName()); Assert.assertTrue(connectorDetails.isConnectorHeaderVisible(connectorForCreate.getName()), "isConnectorTitleVisible()"); navigateToConnectors(); Assert.assertTrue(kafkaConnectList.isConnectorVisible(CONNECTOR_FOR_DELETE.getName()), "isConnectorVisible()"); CONNECTOR_LIST.add(connectorForCreate); } @QaseId(196) @Test public void updateConnector() { navigateToConnectorsAndOpenDetails(CONNECTOR_FOR_UPDATE.getName()); connectorDetails .openConfigTab() .setConfig(CONNECTOR_FOR_UPDATE.getConfig()) .clickSubmitButton(); Assert.assertTrue(connectorDetails.isAlertWithMessageVisible(SUCCESS, "Config successfully updated."), "isAlertWithMessageVisible()"); navigateToConnectors(); Assert.assertTrue(kafkaConnectList.isConnectorVisible(CONNECTOR_FOR_UPDATE.getName()), "isConnectorVisible()"); } @QaseId(195) @Test public void deleteConnector() { navigateToConnectorsAndOpenDetails(CONNECTOR_FOR_DELETE.getName()); connectorDetails .openDotMenu() .clickDeleteBtn() .clickConfirmBtn(); navigateToConnectors(); Assert.assertFalse(kafkaConnectList.isConnectorVisible(CONNECTOR_FOR_DELETE.getName()), "isConnectorVisible()"); CONNECTOR_LIST.remove(CONNECTOR_FOR_DELETE); } @AfterClass(alwaysRun = true) public void afterClass() { CONNECTOR_LIST.forEach(connector -> apiService.deleteConnector(connector.getName())); TOPIC_LIST.forEach(topic -> apiService.deleteTopic(topic.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/ksqldb/KsqlDbTest.java ================================================ package com.provectus.kafka.ui.smokesuite.ksqldb; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlMenuTabs.STREAMS; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SELECT_ALL_FROM; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SHOW_STREAMS; import static com.provectus.kafka.ui.pages.ksqldb.enums.KsqlQueryConfig.SHOW_TABLES; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.pages.ksqldb.models.Stream; import com.provectus.kafka.ui.pages.ksqldb.models.Table; import io.qameta.allure.Step; import io.qase.api.annotation.QaseId; import java.util.ArrayList; import java.util.List; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; public class KsqlDbTest extends BaseTest { private static final Stream DEFAULT_STREAM = new Stream() .setName("DEFAULT_STREAM_" + randomAlphabetic(4).toUpperCase()) .setTopicName("DEFAULT_TOPIC_" + randomAlphabetic(4).toUpperCase()); private static final Table FIRST_TABLE = new Table() .setName("FIRST_TABLE_" + randomAlphabetic(4).toUpperCase()) .setStreamName(DEFAULT_STREAM.getName()); private static final Table SECOND_TABLE = new Table() .setName("SECOND_TABLE_" + randomAlphabetic(4).toUpperCase()) .setStreamName(DEFAULT_STREAM.getName()); private static final List TOPIC_NAMES_LIST = new ArrayList<>(); @BeforeClass(alwaysRun = true) public void beforeClass() { apiService .createStream(DEFAULT_STREAM) .createTables(FIRST_TABLE, SECOND_TABLE); TOPIC_NAMES_LIST.addAll(List.of(DEFAULT_STREAM.getTopicName(), FIRST_TABLE.getName(), SECOND_TABLE.getName())); } @QaseId(284) @Test(priority = 1) public void streamsAndTablesVisibilityCheck() { navigateToKsqlDb(); SoftAssert softly = new SoftAssert(); softly.assertTrue(ksqlDbList.getTableByName(FIRST_TABLE.getName()).isVisible(), "getTableByName()"); softly.assertTrue(ksqlDbList.getTableByName(SECOND_TABLE.getName()).isVisible(), "getTableByName()"); softly.assertAll(); ksqlDbList .openDetailsTab(STREAMS) .waitUntilScreenReady(); Assert.assertTrue(ksqlDbList.getStreamByName(DEFAULT_STREAM.getName()).isVisible(), "getStreamByName()"); } @QaseId(276) @Test(priority = 2) public void clearEnteredQueryCheck() { navigateToKsqlDbAndExecuteRequest(SHOW_TABLES.getQuery()); Assert.assertFalse(ksqlQueryForm.getEnteredQuery().isEmpty(), "getEnteredQuery()"); ksqlQueryForm .clickClearBtn(); Assert.assertTrue(ksqlQueryForm.getEnteredQuery().isEmpty(), "getEnteredQuery()"); } @QaseId(344) @Test(priority = 3) public void clearResultsButtonCheck() { String notValidQuery = "some not valid request"; navigateToKsqlDb(); ksqlDbList .clickExecuteKsqlRequestBtn(); ksqlQueryForm .waitUntilScreenReady() .setQuery(notValidQuery); Assert.assertFalse(ksqlQueryForm.isClearResultsBtnEnabled(), "isClearResultsBtnEnabled()"); ksqlQueryForm .clickExecuteBtn(notValidQuery); Assert.assertFalse(ksqlQueryForm.isClearResultsBtnEnabled(), "isClearResultsBtnEnabled()"); } @QaseId(41) @Test(priority = 4) public void checkShowTablesRequestExecution() { navigateToKsqlDbAndExecuteRequest(SHOW_TABLES.getQuery()); SoftAssert softly = new SoftAssert(); softly.assertTrue(ksqlQueryForm.areResultsVisible(), "areResultsVisible()"); softly.assertTrue(ksqlQueryForm.getItemByName(FIRST_TABLE.getName()).isVisible(), String.format("getItemByName(%s)", FIRST_TABLE.getName())); softly.assertTrue(ksqlQueryForm.getItemByName(SECOND_TABLE.getName()).isVisible(), String.format("getItemByName(%s)", SECOND_TABLE.getName())); softly.assertAll(); } @QaseId(278) @Test(priority = 5) public void checkShowStreamsRequestExecution() { navigateToKsqlDbAndExecuteRequest(SHOW_STREAMS.getQuery()); SoftAssert softly = new SoftAssert(); softly.assertTrue(ksqlQueryForm.areResultsVisible(), "areResultsVisible()"); softly.assertTrue(ksqlQueryForm.getItemByName(DEFAULT_STREAM.getName()).isVisible(), String.format("getItemByName(%s)", FIRST_TABLE.getName())); softly.assertAll(); } @QaseId(86) @Test(priority = 6) public void clearResultsForExecutedRequest() { navigateToKsqlDbAndExecuteRequest(SHOW_TABLES.getQuery()); SoftAssert softly = new SoftAssert(); softly.assertTrue(ksqlQueryForm.areResultsVisible(), "areResultsVisible()"); softly.assertAll(); ksqlQueryForm .clickClearResultsBtn(); softly.assertFalse(ksqlQueryForm.areResultsVisible(), "areResultsVisible()"); softly.assertAll(); } @QaseId(277) @Test(priority = 7) public void stopQueryFunctionalCheck() { navigateToKsqlDbAndExecuteRequest(String.format(SELECT_ALL_FROM.getQuery(), FIRST_TABLE.getName())); Assert.assertTrue(ksqlQueryForm.isAbortBtnVisible(), "isAbortBtnVisible()"); ksqlQueryForm .clickAbortBtn(); Assert.assertTrue(ksqlQueryForm.isCancelledAlertVisible(), "isCancelledAlertVisible()"); } @AfterClass(alwaysRun = true) public void afterClass() { TOPIC_NAMES_LIST.forEach(topicName -> apiService.deleteTopic(topicName)); } @Step private void navigateToKsqlDbAndExecuteRequest(String query) { navigateToKsqlDb(); ksqlDbList .clickExecuteKsqlRequestBtn(); ksqlQueryForm .waitUntilScreenReady() .setQuery(query) .clickExecuteBtn(query); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/schemas/SchemasTest.java ================================================ package com.provectus.kafka.ui.smokesuite.schemas; import static com.provectus.kafka.ui.utilities.FileUtils.fileToString; import com.codeborne.selenide.Condition; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.api.model.CompatibilityLevel; import com.provectus.kafka.ui.models.Schema; import io.qase.api.annotation.QaseId; import java.util.ArrayList; import java.util.List; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; public class SchemasTest extends BaseTest { private static final List SCHEMA_LIST = new ArrayList<>(); private static final Schema AVRO_API = Schema.createSchemaAvro(); private static final Schema JSON_API = Schema.createSchemaJson(); private static final Schema PROTOBUF_API = Schema.createSchemaProtobuf(); @BeforeClass(alwaysRun = true) public void beforeClass() { SCHEMA_LIST.addAll(List.of(AVRO_API, JSON_API, PROTOBUF_API)); SCHEMA_LIST.forEach(schema -> apiService.createSchema(schema)); } @QaseId(43) @Test(priority = 1) public void createSchemaAvro() { Schema schemaAvro = Schema.createSchemaAvro(); navigateToSchemaRegistry(); schemaRegistryList .clickCreateSchema(); schemaCreateForm .setSubjectName(schemaAvro.getName()) .setSchemaField(fileToString(schemaAvro.getValuePath())) .selectSchemaTypeFromDropdown(schemaAvro.getType()) .clickSubmitButton(); schemaDetails .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertTrue(schemaDetails.isSchemaHeaderVisible(schemaAvro.getName()), "isSchemaHeaderVisible()"); softly.assertEquals(schemaDetails.getSchemaType(), schemaAvro.getType().getValue(), "getSchemaType()"); softly.assertEquals(schemaDetails.getCompatibility(), CompatibilityLevel.CompatibilityEnum.BACKWARD.getValue(), "getCompatibility()"); softly.assertAll(); navigateToSchemaRegistry(); Assert.assertTrue(schemaRegistryList.isSchemaVisible(AVRO_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.add(schemaAvro); } @QaseId(186) @Test(priority = 2) public void updateSchemaAvro() { AVRO_API.setValuePath( System.getProperty("user.dir") + "/src/main/resources/testData/schemas/schema_avro_for_update.json"); navigateToSchemaRegistryAndOpenDetails(AVRO_API.getName()); schemaDetails .openEditSchema(); schemaCreateForm .waitUntilScreenReady(); verifyElementsCondition(schemaCreateForm.getAllDetailsPageElements(), Condition.visible); SoftAssert softly = new SoftAssert(); softly.assertFalse(schemaCreateForm.isSubmitBtnEnabled(), "isSubmitBtnEnabled()"); softly.assertFalse(schemaCreateForm.isSchemaDropDownEnabled(), "isSchemaDropDownEnabled()"); softly.assertAll(); schemaCreateForm .selectCompatibilityLevelFromDropdown(CompatibilityLevel.CompatibilityEnum.NONE) .setNewSchemaValue(fileToString(AVRO_API.getValuePath())) .clickSubmitButton(); schemaDetails .waitUntilScreenReady(); Assert.assertEquals(schemaDetails.getCompatibility(), CompatibilityLevel.CompatibilityEnum.NONE.toString(), "getCompatibility()"); } @QaseId(44) @Test(priority = 3) public void compareVersionsOperation() { navigateToSchemaRegistryAndOpenDetails(AVRO_API.getName()); int latestVersion = schemaDetails .waitUntilScreenReady() .getLatestVersion(); schemaDetails .openCompareVersionMenu(); int versionsNumberFromDdl = schemaCreateForm .waitUntilScreenReady() .openLeftVersionDdl() .getVersionsNumberFromList(); Assert.assertEquals(versionsNumberFromDdl, latestVersion, "Versions number is not matched"); schemaCreateForm .selectVersionFromDropDown(1); Assert.assertEquals(schemaCreateForm.getMarkedLinesNumber(), 42, "getAllMarkedLines()"); } @QaseId(187) @Test(priority = 4) public void deleteSchemaAvro() { navigateToSchemaRegistryAndOpenDetails(AVRO_API.getName()); schemaDetails .removeSchema(); schemaRegistryList .waitUntilScreenReady(); Assert.assertFalse(schemaRegistryList.isSchemaVisible(AVRO_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.remove(AVRO_API); } @QaseId(89) @Test(priority = 5) public void createSchemaJson() { Schema schemaJson = Schema.createSchemaJson(); navigateToSchemaRegistry(); schemaRegistryList .clickCreateSchema(); schemaCreateForm .setSubjectName(schemaJson.getName()) .setSchemaField(fileToString(schemaJson.getValuePath())) .selectSchemaTypeFromDropdown(schemaJson.getType()) .clickSubmitButton(); schemaDetails .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertTrue(schemaDetails.isSchemaHeaderVisible(schemaJson.getName()), "isSchemaHeaderVisible()"); softly.assertEquals(schemaDetails.getSchemaType(), schemaJson.getType().getValue(), "getSchemaType()"); softly.assertEquals(schemaDetails.getCompatibility(), CompatibilityLevel.CompatibilityEnum.BACKWARD.getValue(), "getCompatibility()"); softly.assertAll(); navigateToSchemaRegistry(); Assert.assertTrue(schemaRegistryList.isSchemaVisible(JSON_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.add(schemaJson); } @QaseId(189) @Test(priority = 6) public void deleteSchemaJson() { navigateToSchemaRegistryAndOpenDetails(JSON_API.getName()); schemaDetails .removeSchema(); schemaRegistryList .waitUntilScreenReady(); Assert.assertFalse(schemaRegistryList.isSchemaVisible(JSON_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.remove(JSON_API); } @QaseId(91) @Test(priority = 7) public void createSchemaProtobuf() { Schema schemaProtobuf = Schema.createSchemaProtobuf(); navigateToSchemaRegistry(); schemaRegistryList .clickCreateSchema(); schemaCreateForm .setSubjectName(schemaProtobuf.getName()) .setSchemaField(fileToString(schemaProtobuf.getValuePath())) .selectSchemaTypeFromDropdown(schemaProtobuf.getType()) .clickSubmitButton(); schemaDetails .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertTrue(schemaDetails.isSchemaHeaderVisible(schemaProtobuf.getName()), "isSchemaHeaderVisible()"); softly.assertEquals(schemaDetails.getSchemaType(), schemaProtobuf.getType().getValue(), "getSchemaType()"); softly.assertEquals(schemaDetails.getCompatibility(), CompatibilityLevel.CompatibilityEnum.BACKWARD.getValue(), "getCompatibility()"); softly.assertAll(); navigateToSchemaRegistry(); Assert.assertTrue(schemaRegistryList.isSchemaVisible(PROTOBUF_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.add(schemaProtobuf); } @QaseId(223) @Test(priority = 8) public void deleteSchemaProtobuf() { navigateToSchemaRegistryAndOpenDetails(PROTOBUF_API.getName()); schemaDetails .removeSchema(); schemaRegistryList .waitUntilScreenReady(); Assert.assertFalse(schemaRegistryList.isSchemaVisible(PROTOBUF_API.getName()), "isSchemaVisible()"); SCHEMA_LIST.remove(PROTOBUF_API); } @AfterClass(alwaysRun = true) public void afterClass() { SCHEMA_LIST.forEach(schema -> apiService.deleteSchema(schema.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/topics/MessagesTest.java ================================================ package com.provectus.kafka.ui.smokesuite.topics; import static com.provectus.kafka.ui.pages.BasePage.AlertHeader.SUCCESS; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.MESSAGES; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.OVERVIEW; import static com.provectus.kafka.ui.utilities.TimeUtils.waitUntilNewMinuteStarted; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.models.Topic; import io.qameta.allure.Issue; import io.qameta.allure.Step; import io.qase.api.annotation.QaseId; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.stream.IntStream; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Ignore; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; public class MessagesTest extends BaseTest { private static final Topic TOPIC_FOR_MESSAGES = new Topic() .setName("topic-with-clean-message-attribute-" + randomAlphabetic(5)) .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final Topic TOPIC_TO_CLEAR_AND_PURGE_MESSAGES = new Topic() .setName("topic-to-clear-and-purge-messages-" + randomAlphabetic(5)) .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final Topic TOPIC_FOR_CHECK_FILTERS = new Topic() .setName("topic-for-check-filters-" + randomAlphabetic(5)) .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final Topic TOPIC_TO_RECREATE = new Topic() .setName("topic-to-recreate-attribute-" + randomAlphabetic(5)) .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final Topic TOPIC_FOR_CHECK_MESSAGES_COUNT = new Topic() .setName("topic-for-check-messages-count" + randomAlphabetic(5)) .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final List TOPIC_LIST = new ArrayList<>(); @BeforeClass(alwaysRun = true) public void beforeClass() { TOPIC_LIST.addAll(List.of(TOPIC_FOR_MESSAGES, TOPIC_FOR_CHECK_FILTERS, TOPIC_TO_CLEAR_AND_PURGE_MESSAGES, TOPIC_TO_RECREATE, TOPIC_FOR_CHECK_MESSAGES_COUNT)); TOPIC_LIST.forEach(topic -> apiService.createTopic(topic)); IntStream.range(1, 3).forEach(i -> apiService.sendMessage(TOPIC_FOR_CHECK_FILTERS)); waitUntilNewMinuteStarted(); IntStream.range(1, 3).forEach(i -> apiService.sendMessage(TOPIC_FOR_CHECK_FILTERS)); IntStream.range(1, 110).forEach(i -> apiService.sendMessage(TOPIC_FOR_CHECK_MESSAGES_COUNT)); } @QaseId(222) @Test(priority = 1) public void produceMessageCheck() { navigateToTopicsAndOpenDetails(TOPIC_FOR_MESSAGES.getName()); topicDetails .openDetailsTab(MESSAGES); produceMessage(TOPIC_FOR_MESSAGES); Assert.assertEquals(topicDetails.getMessageByKey(TOPIC_FOR_MESSAGES.getMessageKey()).getValue(), TOPIC_FOR_MESSAGES.getMessageValue(), "message.getValue()"); } @QaseId(19) @Test(priority = 2) public void clearMessageCheck() { navigateToTopicsAndOpenDetails(TOPIC_FOR_MESSAGES.getName()); topicDetails .openDetailsTab(OVERVIEW); int messageAmount = topicDetails.getMessageCountAmount(); produceMessage(TOPIC_FOR_MESSAGES); Assert.assertEquals(topicDetails.getMessageCountAmount(), messageAmount + 1, "getMessageCountAmount()"); topicDetails .openDotMenu() .clickClearMessagesMenu() .clickConfirmBtnMdl() .waitUntilScreenReady(); Assert.assertEquals(topicDetails.getMessageCountAmount(), 0, "getMessageCountAmount()"); } @QaseId(239) @Test(priority = 3) public void checkClearTopicMessage() { navigateToTopicsAndOpenDetails(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()); topicDetails .openDetailsTab(OVERVIEW); produceMessage(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES); navigateToTopics(); Assert.assertEquals(topicsList.getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()).getNumberOfMessages(), 1, "getNumberOfMessages()"); topicsList .openDotMenuByTopicName(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()) .clickClearMessagesBtn() .clickConfirmBtnMdl(); SoftAssert softly = new SoftAssert(); softly.assertTrue(topicsList.isAlertWithMessageVisible(SUCCESS, String.format("%s messages have been successfully cleared!", TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName())), "isAlertWithMessageVisible()"); softly.assertEquals(topicsList.getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()).getNumberOfMessages(), 0, "getNumberOfMessages()"); softly.assertAll(); } @QaseId(10) @Test(priority = 4) public void checkPurgeMessagePossibility() { navigateToTopics(); int messageAmount = topicsList.getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()).getNumberOfMessages(); topicsList .openTopic(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()); topicDetails .openDetailsTab(OVERVIEW); produceMessage(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES); navigateToTopics(); Assert.assertEquals(topicsList.getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()).getNumberOfMessages(), messageAmount + 1, "getNumberOfMessages()"); topicsList .getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()) .selectItem(true) .clickPurgeMessagesOfSelectedTopicsBtn(); Assert.assertTrue(topicsList.isConfirmationMdlVisible(), "isConfirmationMdlVisible()"); topicsList .clickCancelBtnMdl() .clickPurgeMessagesOfSelectedTopicsBtn() .clickConfirmBtnMdl(); SoftAssert softly = new SoftAssert(); softly.assertTrue(topicsList.isAlertWithMessageVisible(SUCCESS, String.format("%s messages have been successfully cleared!", TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName())), "isAlertWithMessageVisible()"); softly.assertEquals(topicsList.getTopicItem(TOPIC_TO_CLEAR_AND_PURGE_MESSAGES.getName()).getNumberOfMessages(), 0, "getNumberOfMessages()"); softly.assertAll(); } @QaseId(15) @Test(priority = 6) public void checkMessageFilteringByOffset() { navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); int nextOffset = topicDetails .openDetailsTab(MESSAGES) .getAllMessages().stream() .findFirst().orElseThrow().getOffset() + 1; topicDetails .selectSeekTypeDdlMessagesTab("Offset") .setSeekTypeValueFldMessagesTab(String.valueOf(nextOffset)) .clickSubmitFiltersBtnMessagesTab(); SoftAssert softly = new SoftAssert(); topicDetails.getAllMessages().forEach(message -> softly.assertTrue(message.getOffset() >= nextOffset, String.format("Expected offset not less: %s, but found: %s", nextOffset, message.getOffset()))); softly.assertAll(); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3215") @Issue("https://github.com/provectus/kafka-ui/issues/2345") @QaseId(16) @Test(priority = 7) public void checkMessageFilteringByTimestamp() { navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); LocalDateTime firstTimestamp = topicDetails .openDetailsTab(MESSAGES) .getMessageByOffset(0).getTimestamp(); LocalDateTime nextTimestamp = topicDetails.getAllMessages().stream() .filter(message -> message.getTimestamp().getMinute() != firstTimestamp.getMinute()) .findFirst().orElseThrow().getTimestamp(); topicDetails .selectSeekTypeDdlMessagesTab("Timestamp") .openCalendarSeekType() .selectDateAndTimeByCalendar(nextTimestamp) .clickSubmitFiltersBtnMessagesTab(); SoftAssert softly = new SoftAssert(); topicDetails.getAllMessages().forEach(message -> softly.assertFalse(message.getTimestamp().isBefore(nextTimestamp), String.format("Expected that %s is not before %s.", message.getTimestamp(), nextTimestamp))); softly.assertAll(); } @QaseId(246) @Test(priority = 8) public void checkClearTopicMessageFromOverviewTab() { navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); topicDetails .openDetailsTab(OVERVIEW) .openDotMenu() .clickClearMessagesMenu() .clickConfirmBtnMdl(); SoftAssert softly = new SoftAssert(); softly.assertTrue(topicDetails.isAlertWithMessageVisible(SUCCESS, String.format("%s messages have been successfully cleared!", TOPIC_FOR_CHECK_FILTERS.getName())), "isAlertWithMessageVisible()"); softly.assertEquals(topicDetails.getMessageCountAmount(), 0, "getMessageCountAmount()= " + topicDetails.getMessageCountAmount()); softly.assertAll(); } @QaseId(240) @Test(priority = 9) public void checkRecreateTopic() { navigateToTopicsAndOpenDetails(TOPIC_TO_RECREATE.getName()); topicDetails .openDetailsTab(OVERVIEW); produceMessage(TOPIC_TO_RECREATE); navigateToTopics(); Assert.assertEquals(topicsList.getTopicItem(TOPIC_TO_RECREATE.getName()).getNumberOfMessages(), 1, "getNumberOfMessages()"); topicsList .openDotMenuByTopicName(TOPIC_TO_RECREATE.getName()) .clickRecreateTopicBtn() .clickConfirmBtnMdl(); SoftAssert softly = new SoftAssert(); softly.assertTrue(topicDetails.isAlertWithMessageVisible(SUCCESS, String.format("Topic %s successfully recreated!", TOPIC_TO_RECREATE.getName())), "isAlertWithMessageVisible()"); softly.assertEquals(topicsList.getTopicItem(TOPIC_TO_RECREATE.getName()).getNumberOfMessages(), 0, "getNumberOfMessages()"); softly.assertAll(); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3129") @QaseId(267) @Test(priority = 10) public void checkMessagesCountPerPageWithinTopic() { navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_MESSAGES_COUNT.getName()); topicDetails .openDetailsTab(MESSAGES); int messagesPerPage = topicDetails.getAllMessages().size(); SoftAssert softly = new SoftAssert(); softly.assertEquals(messagesPerPage, 100, "getAllMessages()"); softly.assertFalse(topicDetails.isBackButtonEnabled(), "isBackButtonEnabled()"); softly.assertTrue(topicDetails.isNextButtonEnabled(), "isNextButtonEnabled()"); softly.assertAll(); int lastOffsetOnPage = topicDetails.getAllMessages() .get(messagesPerPage - 1).getOffset(); topicDetails .clickNextButton(); softly.assertEquals(topicDetails.getAllMessages().stream().findFirst().orElseThrow().getOffset(), lastOffsetOnPage + 1, "findFirst().getOffset()"); softly.assertTrue(topicDetails.isBackButtonEnabled(), "isBackButtonEnabled()"); softly.assertFalse(topicDetails.isNextButtonEnabled(), "isNextButtonEnabled()"); softly.assertAll(); } @Step private void produceMessage(Topic topic) { topicDetails .clickProduceMessageBtn(); produceMessagePanel .waitUntilScreenReady() .setKeyField(topic.getMessageKey()) .setValueFiled(topic.getMessageValue()) .submitProduceMessage(); topicDetails .waitUntilScreenReady(); } @AfterClass(alwaysRun = true) public void afterClass() { TOPIC_LIST.forEach(topic -> apiService.deleteTopic(topic.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/java/com/provectus/kafka/ui/smokesuite/topics/TopicsTest.java ================================================ package com.provectus.kafka.ui.smokesuite.topics; import static com.provectus.kafka.ui.pages.BasePage.AlertHeader.SUCCESS; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.CONSUMERS; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.MESSAGES; import static com.provectus.kafka.ui.pages.topics.TopicDetails.TopicMenu.SETTINGS; import static com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue.COMPACT; import static com.provectus.kafka.ui.pages.topics.enums.CleanupPolicyValue.DELETE; import static com.provectus.kafka.ui.pages.topics.enums.CustomParameterType.COMPRESSION_TYPE; import static com.provectus.kafka.ui.pages.topics.enums.MaxSizeOnDisk.NOT_SET; import static com.provectus.kafka.ui.pages.topics.enums.MaxSizeOnDisk.SIZE_1_GB; import static com.provectus.kafka.ui.pages.topics.enums.MaxSizeOnDisk.SIZE_50_GB; import static com.provectus.kafka.ui.pages.topics.enums.TimeToRetain.BTN_2_DAYS; import static com.provectus.kafka.ui.pages.topics.enums.TimeToRetain.BTN_7_DAYS; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import static org.apache.commons.lang3.RandomUtils.nextInt; import com.codeborne.selenide.Condition; import com.provectus.kafka.ui.BaseTest; import com.provectus.kafka.ui.models.Topic; import io.qameta.allure.Issue; import io.qase.api.annotation.QaseId; import java.util.ArrayList; import java.util.List; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Ignore; import org.testng.annotations.Test; import org.testng.asserts.SoftAssert; public class TopicsTest extends BaseTest { private static final Topic TOPIC_TO_CREATE = new Topic() .setName("new-topic-" + randomAlphabetic(5)) .setNumberOfPartitions(1) .setCustomParameterType(COMPRESSION_TYPE) .setCustomParameterValue("producer") .setCleanupPolicyValue(DELETE); private static final Topic TOPIC_TO_UPDATE_AND_DELETE = new Topic() .setName("topic-to-update-and-delete-" + randomAlphabetic(5)) .setNumberOfPartitions(1) .setCleanupPolicyValue(DELETE) .setTimeToRetain(BTN_7_DAYS) .setMaxSizeOnDisk(NOT_SET) .setMaxMessageBytes("1048588") .setMessageKey(randomAlphabetic(5)) .setMessageValue(randomAlphabetic(10)); private static final Topic TOPIC_TO_CHECK_SETTINGS = new Topic() .setName("new-topic-" + randomAlphabetic(5)) .setNumberOfPartitions(1) .setMaxMessageBytes("1000012") .setMaxSizeOnDisk(NOT_SET); private static final Topic TOPIC_FOR_CHECK_FILTERS = new Topic() .setName("topic-for-check-filters-" + randomAlphabetic(5)); private static final Topic TOPIC_FOR_DELETE = new Topic() .setName("topic-to-delete-" + randomAlphabetic(5)); private static final List TOPIC_LIST = new ArrayList<>(); @BeforeClass(alwaysRun = true) public void beforeClass() { TOPIC_LIST.addAll(List.of(TOPIC_TO_UPDATE_AND_DELETE, TOPIC_FOR_DELETE, TOPIC_FOR_CHECK_FILTERS)); TOPIC_LIST.forEach(topic -> apiService.createTopic(topic)); } @QaseId(199) @Test(priority = 1) public void createTopic() { navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(TOPIC_TO_CREATE.getName()) .setNumberOfPartitions(TOPIC_TO_CREATE.getNumberOfPartitions()) .selectCleanupPolicy(TOPIC_TO_CREATE.getCleanupPolicyValue()) .clickSaveTopicBtn(); navigateToTopicsAndOpenDetails(TOPIC_TO_CREATE.getName()); SoftAssert softly = new SoftAssert(); softly.assertTrue(topicDetails.isTopicHeaderVisible(TOPIC_TO_CREATE.getName()), "isTopicHeaderVisible()"); softly.assertEquals(topicDetails.getCleanUpPolicy(), TOPIC_TO_CREATE.getCleanupPolicyValue().toString(), "getCleanUpPolicy()"); softly.assertEquals(topicDetails.getPartitions(), TOPIC_TO_CREATE.getNumberOfPartitions(), "getPartitions()"); softly.assertAll(); navigateToTopics(); Assert.assertTrue(topicsList.isTopicVisible(TOPIC_TO_CREATE.getName()), "isTopicVisible()"); TOPIC_LIST.add(TOPIC_TO_CREATE); } @QaseId(7) @Test(priority = 2) void checkAvailableOperations() { navigateToTopics(); topicsList .getTopicItem(TOPIC_TO_UPDATE_AND_DELETE.getName()) .selectItem(true); verifyElementsCondition(topicsList.getActionButtons(), Condition.enabled); topicsList .getTopicItem(TOPIC_FOR_CHECK_FILTERS.getName()) .selectItem(true); Assert.assertFalse(topicsList.isCopySelectedTopicBtnEnabled(), "isCopySelectedTopicBtnEnabled()"); } @Ignore @Issue("https://github.com/provectus/kafka-ui/issues/3071") @QaseId(268) @Test(priority = 3) public void checkCustomParametersWithinEditExistingTopic() { navigateToTopicsAndOpenDetails(TOPIC_TO_UPDATE_AND_DELETE.getName()); topicDetails .openDotMenu() .clickEditSettingsMenu(); SoftAssert softly = new SoftAssert(); topicCreateEditForm .waitUntilScreenReady() .clickAddCustomParameterTypeButton() .openCustomParameterTypeDdl() .getAllDdlOptions() .forEach(option -> softly.assertTrue(!option.is(Condition.attribute("disabled")), option.getText() + " is enabled:")); softly.assertAll(); } @QaseId(197) @Test(priority = 4) public void updateTopic() { navigateToTopicsAndOpenDetails(TOPIC_TO_UPDATE_AND_DELETE.getName()); topicDetails .openDotMenu() .clickEditSettingsMenu(); topicCreateEditForm .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertEquals(topicCreateEditForm.getCleanupPolicy(), TOPIC_TO_UPDATE_AND_DELETE.getCleanupPolicyValue().getVisibleText(), "getCleanupPolicy()"); softly.assertEquals(topicCreateEditForm.getTimeToRetain(), TOPIC_TO_UPDATE_AND_DELETE.getTimeToRetain().getValue(), "getTimeToRetain()"); softly.assertEquals(topicCreateEditForm.getMaxSizeOnDisk(), TOPIC_TO_UPDATE_AND_DELETE.getMaxSizeOnDisk().getVisibleText(), "getMaxSizeOnDisk()"); softly.assertEquals(topicCreateEditForm.getMaxMessageBytes(), TOPIC_TO_UPDATE_AND_DELETE.getMaxMessageBytes(), "getMaxMessageBytes()"); softly.assertAll(); TOPIC_TO_UPDATE_AND_DELETE .setCleanupPolicyValue(COMPACT) .setTimeToRetain(BTN_2_DAYS) .setMaxSizeOnDisk(SIZE_50_GB).setMaxMessageBytes("1048589"); topicCreateEditForm .selectCleanupPolicy((TOPIC_TO_UPDATE_AND_DELETE.getCleanupPolicyValue())) .setTimeToRetainDataByButtons(TOPIC_TO_UPDATE_AND_DELETE.getTimeToRetain()) .setMaxSizeOnDiskInGB(TOPIC_TO_UPDATE_AND_DELETE.getMaxSizeOnDisk()) .setMaxMessageBytes(TOPIC_TO_UPDATE_AND_DELETE.getMaxMessageBytes()) .clickSaveTopicBtn(); softly.assertTrue(topicDetails.isAlertWithMessageVisible(SUCCESS, "Topic successfully updated."), "isAlertWithMessageVisible()"); softly.assertTrue(topicDetails.isTopicHeaderVisible(TOPIC_TO_UPDATE_AND_DELETE.getName()), "isTopicHeaderVisible()"); softly.assertAll(); topicDetails .waitUntilScreenReady(); navigateToTopicsAndOpenDetails(TOPIC_TO_UPDATE_AND_DELETE.getName()); topicDetails .openDotMenu() .clickEditSettingsMenu(); softly.assertFalse(topicCreateEditForm.isNameFieldEnabled(), "isNameFieldEnabled()"); softly.assertEquals(topicCreateEditForm.getCleanupPolicy(), TOPIC_TO_UPDATE_AND_DELETE.getCleanupPolicyValue().getVisibleText(), "getCleanupPolicy()"); softly.assertEquals(topicCreateEditForm.getTimeToRetain(), TOPIC_TO_UPDATE_AND_DELETE.getTimeToRetain().getValue(), "getTimeToRetain()"); softly.assertEquals(topicCreateEditForm.getMaxSizeOnDisk(), TOPIC_TO_UPDATE_AND_DELETE.getMaxSizeOnDisk().getVisibleText(), "getMaxSizeOnDisk()"); softly.assertEquals(topicCreateEditForm.getMaxMessageBytes(), TOPIC_TO_UPDATE_AND_DELETE.getMaxMessageBytes(), "getMaxMessageBytes()"); softly.assertAll(); } @QaseId(242) @Test(priority = 5) public void removeTopicFromTopicList() { navigateToTopics(); topicsList .openDotMenuByTopicName(TOPIC_TO_UPDATE_AND_DELETE.getName()) .clickRemoveTopicBtn() .clickConfirmBtnMdl(); Assert.assertTrue(topicsList.isAlertWithMessageVisible(SUCCESS, String.format("Topic %s successfully deleted!", TOPIC_TO_UPDATE_AND_DELETE.getName())), "isAlertWithMessageVisible()"); TOPIC_LIST.remove(TOPIC_TO_UPDATE_AND_DELETE); } @QaseId(207) @Test(priority = 6) public void deleteTopic() { navigateToTopicsAndOpenDetails(TOPIC_FOR_DELETE.getName()); topicDetails .openDotMenu() .clickDeleteTopicMenu() .clickConfirmBtnMdl(); navigateToTopics(); Assert.assertFalse(topicsList.isTopicVisible(TOPIC_FOR_DELETE.getName()), "isTopicVisible"); TOPIC_LIST.remove(TOPIC_FOR_DELETE); } @QaseId(20) @Test(priority = 7) public void redirectToConsumerFromTopic() { String topicName = "source-activities"; String consumerGroupId = "connect-sink_postgres_activities"; navigateToTopicsAndOpenDetails(topicName); topicDetails .openDetailsTab(CONSUMERS) .openConsumerGroup(consumerGroupId); consumersDetails .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertTrue(consumersDetails.isRedirectedConsumerTitleVisible(consumerGroupId), "isRedirectedConsumerTitleVisible()"); softly.assertTrue(consumersDetails.isTopicInConsumersDetailsVisible(topicName), "isTopicInConsumersDetailsVisible()"); softly.assertAll(); } @QaseId(4) @Test(priority = 8) public void checkTopicCreatePossibility() { navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady(); Assert.assertFalse(topicCreateEditForm.isCreateTopicButtonEnabled(), "isCreateTopicButtonEnabled()"); topicCreateEditForm .setTopicName("testName"); Assert.assertFalse(topicCreateEditForm.isCreateTopicButtonEnabled(), "isCreateTopicButtonEnabled()"); topicCreateEditForm .setTopicName(null) .setNumberOfPartitions(nextInt(1, 10)); Assert.assertFalse(topicCreateEditForm.isCreateTopicButtonEnabled(), "isCreateTopicButtonEnabled()"); topicCreateEditForm .setTopicName("testName"); Assert.assertTrue(topicCreateEditForm.isCreateTopicButtonEnabled(), "isCreateTopicButtonEnabled()"); } @QaseId(266) @Test(priority = 9) public void checkTimeToRetainDataCustomValueWithEditingTopic() { Topic topicToRetainData = new Topic() .setName("topic-to-retain-data-" + randomAlphabetic(5)) .setTimeToRetainData("86400000"); navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(topicToRetainData.getName()) .setNumberOfPartitions(1) .setTimeToRetainDataInMs("604800000"); Assert.assertEquals(topicCreateEditForm.getTimeToRetain(), "604800000", "getTimeToRetain()"); topicCreateEditForm .setTimeToRetainDataInMs(topicToRetainData.getTimeToRetainData()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady() .openDotMenu() .clickEditSettingsMenu(); Assert.assertEquals(topicCreateEditForm.getTimeToRetain(), topicToRetainData.getTimeToRetainData(), "getTimeToRetain()"); topicDetails .openDetailsTab(SETTINGS); Assert.assertEquals(topicDetails.getSettingsGridValueByKey("retention.ms"), topicToRetainData.getTimeToRetainData(), "getSettingsGridValueByKey()"); TOPIC_LIST.add(topicToRetainData); } @QaseId(6) @Test(priority = 10) public void checkCustomParametersWithinCreateNewTopic() { navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(TOPIC_TO_CREATE.getName()) .clickAddCustomParameterTypeButton() .setCustomParameterType(TOPIC_TO_CREATE.getCustomParameterType()); Assert.assertTrue(topicCreateEditForm.isDeleteCustomParameterButtonEnabled(), "isDeleteCustomParameterButtonEnabled()"); topicCreateEditForm .clearCustomParameterValue(); Assert.assertTrue(topicCreateEditForm.isValidationMessageCustomParameterValueVisible(), "isValidationMessageCustomParameterValueVisible()"); } @QaseId(2) @Test(priority = 11) public void checkTopicListElements() { navigateToTopics(); verifyElementsCondition(topicsList.getAllVisibleElements(), Condition.visible); verifyElementsCondition(topicsList.getAllEnabledElements(), Condition.enabled); } @QaseId(12) @Test(priority = 12) public void addNewFilterWithinTopic() { String filterName = randomAlphabetic(5); navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); topicDetails .openDetailsTab(MESSAGES) .clickMessagesAddFiltersBtn() .waitUntilAddFiltersMdlVisible(); verifyElementsCondition(topicDetails.getAllAddFilterModalVisibleElements(), Condition.visible); verifyElementsCondition(topicDetails.getAllAddFilterModalEnabledElements(), Condition.enabled); verifyElementsCondition(topicDetails.getAllAddFilterModalDisabledElements(), Condition.disabled); Assert.assertFalse(topicDetails.isSaveThisFilterCheckBoxSelected(), "isSaveThisFilterCheckBoxSelected()"); topicDetails .setFilterCodeFldAddFilterMdl(filterName); Assert.assertTrue(topicDetails.isAddFilterBtnAddFilterMdlEnabled(), "isAddFilterBtnAddFilterMdlEnabled()"); topicDetails.clickAddFilterBtnAndCloseMdl(true); Assert.assertTrue(topicDetails.isActiveFilterVisible(filterName), "isActiveFilterVisible()"); } @QaseId(352) @Test(priority = 13) public void editActiveSmartFilterCheck() { String filterName = randomAlphabetic(5); String filterCode = randomAlphabetic(5); navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); topicDetails .openDetailsTab(MESSAGES) .clickMessagesAddFiltersBtn() .waitUntilAddFiltersMdlVisible() .setFilterCodeFldAddFilterMdl(filterCode) .setDisplayNameFldAddFilterMdl(filterName) .clickAddFilterBtnAndCloseMdl(true) .clickEditActiveFilterBtn(filterName) .waitUntilAddFiltersMdlVisible(); SoftAssert softly = new SoftAssert(); softly.assertEquals(topicDetails.getFilterCodeValue(), filterCode, "getFilterCodeValue()"); softly.assertEquals(topicDetails.getFilterNameValue(), filterName, "getFilterNameValue()"); softly.assertAll(); String newFilterName = randomAlphabetic(5); String newFilterCode = randomAlphabetic(5); topicDetails .setFilterCodeFldAddFilterMdl(newFilterCode) .setDisplayNameFldAddFilterMdl(newFilterName) .clickSaveFilterBtnAndCloseMdl(true); softly.assertTrue(topicDetails.isActiveFilterVisible(newFilterName), "isActiveFilterVisible()"); softly.assertEquals(topicDetails.getSearchFieldValue(), newFilterCode, "getSearchFieldValue()"); softly.assertAll(); } @QaseId(13) @Test(priority = 14) public void checkFilterSavingWithinSavedFilters() { String displayName = randomAlphabetic(5); navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); topicDetails .openDetailsTab(MESSAGES) .clickMessagesAddFiltersBtn() .waitUntilAddFiltersMdlVisible() .setFilterCodeFldAddFilterMdl(randomAlphabetic(4)) .selectSaveThisFilterCheckboxMdl(true) .setDisplayNameFldAddFilterMdl(displayName); Assert.assertTrue(topicDetails.isAddFilterBtnAddFilterMdlEnabled(), "isAddFilterBtnAddFilterMdlEnabled()"); topicDetails .clickAddFilterBtnAndCloseMdl(false) .openSavedFiltersListMdl(); Assert.assertTrue(topicDetails.isFilterVisibleAtSavedFiltersMdl(displayName), "isFilterVisibleAtSavedFiltersMdl()"); } @QaseId(14) @Test(priority = 15) public void checkApplyingSavedFilterWithinTopicMessages() { String displayName = randomAlphabetic(5); navigateToTopicsAndOpenDetails(TOPIC_FOR_CHECK_FILTERS.getName()); topicDetails .openDetailsTab(MESSAGES) .clickMessagesAddFiltersBtn() .waitUntilAddFiltersMdlVisible() .setFilterCodeFldAddFilterMdl(randomAlphabetic(4)) .selectSaveThisFilterCheckboxMdl(true) .setDisplayNameFldAddFilterMdl(displayName) .clickAddFilterBtnAndCloseMdl(false) .openSavedFiltersListMdl() .selectFilterAtSavedFiltersMdl(displayName) .clickSelectFilterBtnAtSavedFiltersMdl(); Assert.assertTrue(topicDetails.isActiveFilterVisible(displayName), "isActiveFilterVisible()"); } @QaseId(11) @Test(priority = 16) public void checkShowInternalTopicsButton() { navigateToTopics(); topicsList .setShowInternalRadioButton(true); Assert.assertTrue(topicsList.getInternalTopics().size() > 0, "getInternalTopics()"); topicsList .goToLastPage(); Assert.assertTrue(topicsList.getNonInternalTopics().size() > 0, "getNonInternalTopics()"); topicsList .setShowInternalRadioButton(false); SoftAssert softly = new SoftAssert(); softly.assertEquals(topicsList.getInternalTopics().size(), 0, "getInternalTopics()"); softly.assertTrue(topicsList.getNonInternalTopics().size() > 0, "getNonInternalTopics()"); softly.assertAll(); } @QaseId(334) @Test(priority = 17) public void checkInternalTopicsNaming() { navigateToTopics(); SoftAssert softly = new SoftAssert(); topicsList .setShowInternalRadioButton(true) .getInternalTopics() .forEach(topic -> softly.assertTrue(topic.getName().startsWith("_"), String.format("'%s' starts with '_'", topic.getName()))); softly.assertAll(); } @QaseId(56) @Test(priority = 18) public void checkRetentionBytesAccordingToMaxSizeOnDisk() { navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(TOPIC_TO_CHECK_SETTINGS.getName()) .setNumberOfPartitions(TOPIC_TO_CHECK_SETTINGS.getNumberOfPartitions()) .setMaxMessageBytes(TOPIC_TO_CHECK_SETTINGS.getMaxMessageBytes()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady(); TOPIC_LIST.add(TOPIC_TO_CHECK_SETTINGS); topicDetails .openDetailsTab(SETTINGS); topicSettingsTab .waitUntilScreenReady(); SoftAssert softly = new SoftAssert(); softly.assertEquals(topicSettingsTab.getValueByKey("retention.bytes"), TOPIC_TO_CHECK_SETTINGS.getMaxSizeOnDisk().getOptionValue(), "getValueOfKey(retention.bytes)"); softly.assertEquals(topicSettingsTab.getValueByKey("max.message.bytes"), TOPIC_TO_CHECK_SETTINGS.getMaxMessageBytes(), "getValueOfKey(max.message.bytes)"); softly.assertAll(); TOPIC_TO_CHECK_SETTINGS .setMaxSizeOnDisk(SIZE_1_GB) .setMaxMessageBytes("1000056"); topicDetails .openDotMenu() .clickEditSettingsMenu(); topicCreateEditForm .waitUntilScreenReady() .setMaxSizeOnDiskInGB(TOPIC_TO_CHECK_SETTINGS.getMaxSizeOnDisk()) .setMaxMessageBytes(TOPIC_TO_CHECK_SETTINGS.getMaxMessageBytes()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady() .openDetailsTab(SETTINGS); topicSettingsTab .waitUntilScreenReady(); softly.assertEquals(topicSettingsTab.getValueByKey("retention.bytes"), TOPIC_TO_CHECK_SETTINGS.getMaxSizeOnDisk().getOptionValue(), "getValueOfKey(retention.bytes)"); softly.assertEquals(topicSettingsTab.getValueByKey("max.message.bytes"), TOPIC_TO_CHECK_SETTINGS.getMaxMessageBytes(), "getValueOfKey(max.message.bytes)"); softly.assertAll(); } @QaseId(247) @Test(priority = 19) public void recreateTopicFromTopicProfile() { Topic topicToRecreate = new Topic() .setName("topic-to-recreate-" + randomAlphabetic(5)) .setNumberOfPartitions(1); navigateToTopics(); topicsList .clickAddTopicBtn(); topicCreateEditForm .waitUntilScreenReady() .setTopicName(topicToRecreate.getName()) .setNumberOfPartitions(topicToRecreate.getNumberOfPartitions()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady(); TOPIC_LIST.add(topicToRecreate); topicDetails .openDotMenu() .clickRecreateTopicMenu(); Assert.assertTrue(topicDetails.isConfirmationMdlVisible(), "isConfirmationMdlVisible()"); topicDetails .clickConfirmBtnMdl(); Assert.assertTrue(topicDetails.isAlertWithMessageVisible(SUCCESS, String.format("Topic %s successfully recreated!", topicToRecreate.getName())), "isAlertWithMessageVisible()"); } @QaseId(8) @Test(priority = 20) public void checkCopyTopicPossibility() { Topic topicToCopy = new Topic() .setName("topic-to-copy-" + randomAlphabetic(5)) .setNumberOfPartitions(1); navigateToTopics(); topicsList .getAnyNonInternalTopic() .selectItem(true) .clickCopySelectedTopicBtn(); topicCreateEditForm .waitUntilScreenReady(); Assert.assertFalse(topicCreateEditForm.isCreateTopicButtonEnabled(), "isCreateTopicButtonEnabled()"); topicCreateEditForm .setTopicName(topicToCopy.getName()) .setNumberOfPartitions(topicToCopy.getNumberOfPartitions()) .clickSaveTopicBtn(); topicDetails .waitUntilScreenReady(); TOPIC_LIST.add(topicToCopy); Assert.assertTrue(topicDetails.isTopicHeaderVisible(topicToCopy.getName()), "isTopicHeaderVisible()"); } @AfterClass(alwaysRun = true) public void afterClass() { TOPIC_LIST.forEach(topic -> apiService.deleteTopic(topic.getName())); } } ================================================ FILE: kafka-ui-e2e-checks/src/test/resources/manual.xml ================================================ ================================================ FILE: kafka-ui-e2e-checks/src/test/resources/qase.xml ================================================ ================================================ FILE: kafka-ui-e2e-checks/src/test/resources/regression.xml ================================================ ================================================ FILE: kafka-ui-e2e-checks/src/test/resources/sanity.xml ================================================ ================================================ FILE: kafka-ui-e2e-checks/src/test/resources/smoke.xml ================================================ ================================================ FILE: kafka-ui-react-app/.editorconfig ================================================ root = true [*] end_of_line = lf indent_style = space indent_size = 2 charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true ================================================ FILE: kafka-ui-react-app/.eslintignore ================================================ /src/generated-sources/** ================================================ FILE: kafka-ui-react-app/.eslintrc.json ================================================ { "env": { "browser": true, "es6": true, "jest": true }, "globals": { "Atomics": "readonly", "SharedArrayBuffer": "readonly" }, "parser": "@typescript-eslint/parser", "parserOptions": { "ecmaFeatures": { "jsx": true }, "ecmaVersion": 2018, "sourceType": "module", "project": [ "./tsconfig.json", "./src/setupTests.ts" ] }, "plugins": [ "react", "@typescript-eslint", "prettier", "react-hooks" ], "extends": [ "airbnb", "airbnb-typescript", "plugin:@typescript-eslint/recommended", "plugin:jest-dom/recommended", "plugin:prettier/recommended", "eslint:recommended", "plugin:react/recommended", "prettier" ], "rules": { "react/no-unused-prop-types": "off", "react/require-default-props": "off", "prettier/prettier": "warn", "@typescript-eslint/explicit-module-boundary-types": "off", "jsx-a11y/label-has-associated-control": "off", "import/prefer-default-export": "off", "@typescript-eslint/no-explicit-any": "error", "react-hooks/rules-of-hooks": "error", // Checks rules of Hooks // breaks builds as we still have those warns "react-hooks/exhaustive-deps": "off", // Checks effect dependencies "import/no-extraneous-dependencies": [ "error", { "devDependencies": true } ], "import/no-cycle": "error", "import/order": [ "error", { "groups": [ "builtin", "external", "parent", "sibling", "index" ], "newlines-between": "always" } ], "import/no-relative-parent-imports": "error", "no-debugger": "warn", "react/jsx-props-no-spreading": "off", "no-param-reassign": [ "error", { "props": true, "ignorePropertyModificationsFor": [ "state" ] } ], "react/function-component-definition": [ 2, { "namedComponents": "arrow-function", "unnamedComponents": "arrow-function" } ], "react/jsx-no-constructed-context-values": "off", "react/display-name": "off" }, "overrides": [ { "files": [ "**/*.tsx" ], "rules": { "react/prop-types": "off" } }, { "files": [ "*.spec.tsx" ], "rules": { "react/jsx-props-no-spreading": "off" } } ] } ================================================ FILE: kafka-ui-react-app/.gitignore ================================================ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. # dependencies node_modules .pnp .pnp.js node package-lock.json # testing coverage # production build # misc .DS_Store .env.local .env.development.local .env.test.local .env.production.local pnpm-debug.log* npm-debug.log* yarn-debug.log* yarn-error.log* .idea # generated sources src/generated-sources .eslintcache ================================================ FILE: kafka-ui-react-app/.jest/cssTransform.js ================================================ 'use strict'; // This is a custom Jest transformer turning style imports into empty objects. // http://facebook.github.io/jest/docs/en/webpack.html module.exports = { process() { return { code: 'module.exports = {};', }; }, getCacheKey() { // The output is always the same. return 'cssTransform'; }, }; ================================================ FILE: kafka-ui-react-app/.jest/resolver.js ================================================ module.exports = (path, options) => { // Call the defaultResolver, so we leverage its cache, error handling, etc. return options.defaultResolver(path, { ...options, // Use packageFilter to process parsed `package.json` before // the resolution (see https://www.npmjs.com/package/resolve#resolveid-opts-cb) packageFilter: (pkg) => { // jest-environment-jsdom 28+ tries to use browser exports instead of default exports, // but @hookform/resolvers only offers an ESM browser export and not a CommonJS one. Jest does not yet // support ESM modules natively, so this causes a Jest error related to trying to parse // "export" syntax. // // This workaround prevents Jest from considering @hookform/resolvers module-based exports at all; // it falls back to CommonJS+node "main" property. if (pkg.name === '@hookform/resolvers') { delete pkg['exports']; delete pkg['module']; } if (pkg.name === 'jsonpath-plus') { delete pkg['exports']; delete pkg['module']; } return pkg; }, }); }; ================================================ FILE: kafka-ui-react-app/.nvmrc ================================================ v18.17.1 ================================================ FILE: kafka-ui-react-app/.prettierrc ================================================ { "trailingComma": "es5", "semi": true, "singleQuote": true, "quoteProps": "as-needed", "jsxSingleQuote": false, "bracketSpacing": true, "bracketSameLine": false, "arrowParens": "always" } ================================================ FILE: kafka-ui-react-app/README.md ================================================ # UI for Apache Kafka UI for Apache Kafka management [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=com.provectus%3Akafka-ui_frontend&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=com.provectus%3Akafka-ui_frontend) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=com.provectus%3Akafka-ui_frontend&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=com.provectus%3Akafka-ui_frontend) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=com.provectus%3Akafka-ui_frontend&metric=coverage)](https://sonarcloud.io/summary/new_code?id=com.provectus%3Akafka-ui_frontend) ## Table of contents - [Requirements](#requirements) - [Getting started](#getting-started) - [Links](#links) ## Requirements - [docker](https://www.docker.com/get-started) (required to run [Initialize application](#initialize-application)) - [nvm](https://github.com/nvm-sh/nvm) with installed [Node.js](https://nodejs.org/en/) of expected version (check `.nvmrc`) ## Getting started Go to react app folder ```sh cd ./kafka-ui-react-app ``` Install [pnpm](https://pnpm.io/installation) ``` npm install -g pnpm ``` Install dependencies ``` pnpm install ``` Generate API clients from OpenAPI document ```sh pnpm gen:sources ``` ## Start application ### Proxying API Requests in Development Create or update existing `.env.local` file with ``` VITE_DEV_PROXY= https://api.server # your API server ``` Run the application ```sh pnpm dev ``` ### Docker way Have to be run from root directory. Start UI for Apache Kafka with your Kafka clusters: ```sh docker-compose -f ./documentation/compose/kafka-ui.yaml up ``` Make sure that none of the `.env*` files contain `DEV_PROXY` variable Run the application ```sh pnpm dev ``` ## Links * [Vite](https://github.com/vitejs/vite) ================================================ FILE: kafka-ui-react-app/index.html ================================================ UI for Apache Kafka
================================================ FILE: kafka-ui-react-app/jest.config.ts ================================================ import type { Config } from '@jest/types'; export default { roots: ['/src'], collectCoverageFrom: ['src/**/*.{js,jsx,ts,tsx}', '!src/**/*.d.ts'], coveragePathIgnorePatterns: [ '/node_modules/', '/src/generated-sources/', '/src/lib/fixtures/', '/vite.config.ts', '/src/index.tsx', '/src/serviceWorker.ts', ], coverageReporters: ['json', 'lcov', 'text', 'clover'], resolver: '/.jest/resolver.js', setupFilesAfterEnv: ['/src/setupTests.ts'], testMatch: [ '/src/**/__{test,tests}__/**/*.{spec,test}.{js,jsx,ts,tsx}', ], testEnvironment: 'jsdom', transform: { '\\.[jt]sx?$': '@swc/jest', '^.+\\.css$': '/.jest/cssTransform.js', }, transformIgnorePatterns: [ '[/\\\\]node_modules[/\\\\].+\\.(js|jsx|mjs|cjs|ts|tsx)$', '^.+\\.module\\.(css|sass|scss)$', ], modulePaths: ['/src'], watchPlugins: [ 'jest-watch-typeahead/filename', 'jest-watch-typeahead/testname', ], resetMocks: true, reporters: ['default', 'github-actions'], } as Config.InitialOptions; ================================================ FILE: kafka-ui-react-app/openapitools.json ================================================ { "$schema": "node_modules/@openapitools/openapi-generator-cli/config.schema.json", "spaces": 2, "generator-cli": { "version": "5.3.0", "generators": { "fetch": { "generatorName": "typescript-fetch", "output": "src/generated-sources", "glob": "../kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml", "additionalProperties": { "enumPropertyNaming": "UPPERCASE", "typescriptThreePlus": true, "supportsES6": true, "nullSafeAdditionalProps": true, "withInterfaces": true }, "typeMappings": { "object": "any" } } } } } ================================================ FILE: kafka-ui-react-app/package.json ================================================ { "name": "kafka-ui", "version": "0.4.0", "homepage": "./", "private": true, "dependencies": { "@floating-ui/react": "^0.19.2", "@hookform/error-message": "^2.0.0", "@hookform/resolvers": "^2.7.1", "@microsoft/fetch-event-source": "^2.0.1", "@reduxjs/toolkit": "^1.8.3", "@szhsin/react-menu": "^3.5.3", "@tanstack/react-query": "^4.0.5", "@tanstack/react-table": "^8.5.10", "@testing-library/react": "^14.0.0", "@types/testing-library__jest-dom": "^5.14.5", "ace-builds": "^1.7.1", "ajv": "^8.6.3", "ajv-formats": "^2.1.1", "classnames": "^2.2.6", "fetch-mock": "^9.11.0", "jest": "^29.4.3", "jest-watch-typeahead": "^2.2.2", "json-schema-faker": "^0.5.0-rcv.44", "jsonpath-plus": "^7.2.0", "lodash": "^4.17.21", "lossless-json": "^2.0.8", "pretty-ms": "7.0.1", "react": "^18.1.0", "react-ace": "^10.1.0", "react-datepicker": "^4.10.0", "react-dom": "^18.1.0", "react-error-boundary": "^3.1.4", "react-hook-form": "7.43.1", "react-hot-toast": "^2.4.0", "react-is": "^18.2.0", "react-multi-select-component": "^4.3.3", "react-redux": "^8.0.2", "react-router-dom": "^6.3.0", "redux": "^4.2.0", "sass": "^1.52.3", "styled-components": "^5.3.1", "use-debounce": "^9.0.3", "vite": "^4.0.0", "vite-tsconfig-paths": "^4.0.2", "whatwg-fetch": "^3.6.2", "yup": "^1.0.0", "zustand": "^4.1.1" }, "scripts": { "start": "vite", "dev": "vite", "gen:sources": "rimraf src/generated-sources && openapi-generator-cli generate", "build": "vite build", "preview": "vite preview", "lint": "eslint --ext .tsx,.ts src/", "lint:fix": "eslint --ext .tsx,.ts src/ --fix", "lint:CI": "eslint --ext .tsx,.ts src/ --max-warnings=0", "test": "jest --watch", "test:coverage": "jest --watchAll --coverage", "test:CI": "CI=true pnpm test:coverage --ci --testResultsProcessor=\"jest-sonar-reporter\" --watchAll=false", "tsc": "tsc --pretty --noEmit", "deadcode": "ts-prune -i src/generated-sources" }, "devDependencies": { "@jest/types": "^29.4.3", "@openapitools/openapi-generator-cli": "^2.5.2", "@swc/core": "^1.3.36", "@swc/jest": "^0.2.24", "@testing-library/dom": "^9.0.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/user-event": "^14.4.3", "@types/eventsource": "^1.1.8", "@types/lodash": "^4.14.172", "@types/lossless-json": "^1.0.1", "@types/node": "^16.4.13", "@types/react": "^18.0.9", "@types/react-datepicker": "^4.8.0", "@types/react-dom": "^18.0.3", "@types/react-router-dom": "^5.3.3", "@types/styled-components": "^5.1.13", "@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/parser": "^5.29.0", "@vitejs/plugin-react-swc": "^3.0.0", "dotenv": "^16.0.1", "eslint": "^8.3.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^17.0.0", "eslint-config-prettier": "^9.0.0", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.2.7", "eslint-plugin-import": "^2.26.0", "eslint-plugin-jest-dom": "^4.0.3", "eslint-plugin-jsx-a11y": "^6.5.1", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-react": "^7.30.1", "eslint-plugin-react-hooks": "^4.5.0", "jest-environment-jsdom": "^29.4.3", "jest-sonar-reporter": "^2.0.0", "jest-styled-components": "^7.1.1", "prettier": "^2.8.4", "rimraf": "^4.1.2", "ts-node": "^10.9.1", "ts-prune": "^0.10.3", "typescript": "^4.7.4", "vite-plugin-ejs": "^1.6.4" }, "engines": { "node": "v18.17.1", "pnpm": "^8.6.12" } } ================================================ FILE: kafka-ui-react-app/public/manifest.json ================================================ { "name": "UI for Apache Kafka", "icons": [ { "src": "/favicon/icon-192.png", "type": "image/png", "sizes": "192x192" }, { "src": "/favicon/icon-512.png", "type": "image/png", "sizes": "512x512" } ] } ================================================ FILE: kafka-ui-react-app/public/robots.txt ================================================ User-agent: * Disallow: / ================================================ FILE: kafka-ui-react-app/sonar-project.properties ================================================ sonar.projectKey=com.provectus:kafka-ui_frontend sonar.organization=provectus sonar.sources=. sonar.exclusions=**/__tests__/**,**/__test__/**,src/serviceWorker.ts,src/setupTests.ts,src/setupProxy.js,**/fixtures.ts,src/lib/fixtures/**,src/lib/testHelpers.tsx,src/index.tsx,vite.config.ts,config/** sonar.typescript.lcov.reportPaths=./coverage/lcov.info sonar.testExecutionReportPaths=./test-report.xml sonar.sourceEncoding=UTF-8 ================================================ FILE: kafka-ui-react-app/src/components/ACLPage/ACLPage.tsx ================================================ import React from 'react'; import { Routes, Route } from 'react-router-dom'; import ACList from 'components/ACLPage/List/List'; const ACLPage = () => { return ( } /> ); }; export default ACLPage; ================================================ FILE: kafka-ui-react-app/src/components/ACLPage/List/List.styled.ts ================================================ import styled from 'styled-components'; export const EnumCell = styled.div` text-transform: capitalize; `; export const DeleteCell = styled.div` svg { cursor: pointer; } `; export const Chip = styled.div<{ chipType?: 'default' | 'success' | 'danger' | 'secondary' | string; }>` width: fit-content; text-transform: capitalize; padding: 2px 8px; font-size: 12px; line-height: 16px; border-radius: 16px; color: ${({ theme }) => theme.tag.color}; background-color: ${({ theme, chipType }) => { switch (chipType) { case 'success': return theme.tag.backgroundColor.green; case 'danger': return theme.tag.backgroundColor.red; case 'secondary': return theme.tag.backgroundColor.secondary; default: return theme.tag.backgroundColor.gray; } }}; `; export const PatternCell = styled.div` display: flex; align-items: center; ${Chip} { margin-left: 4px; } `; ================================================ FILE: kafka-ui-react-app/src/components/ACLPage/List/List.tsx ================================================ import React from 'react'; import { ColumnDef } from '@tanstack/react-table'; import { useTheme } from 'styled-components'; import PageHeading from 'components/common/PageHeading/PageHeading'; import Table from 'components/common/NewTable'; import DeleteIcon from 'components/common/Icons/DeleteIcon'; import { useConfirm } from 'lib/hooks/useConfirm'; import useAppParams from 'lib/hooks/useAppParams'; import { useAcls, useDeleteAcl } from 'lib/hooks/api/acl'; import { ClusterName } from 'redux/interfaces'; import { KafkaAcl, KafkaAclNamePatternType, KafkaAclPermissionEnum, } from 'generated-sources'; import * as S from './List.styled'; const ACList: React.FC = () => { const { clusterName } = useAppParams<{ clusterName: ClusterName }>(); const theme = useTheme(); const { data: aclList } = useAcls(clusterName); const { deleteResource } = useDeleteAcl(clusterName); const modal = useConfirm(true); const [rowId, setRowId] = React.useState(''); const onDeleteClick = (acl: KafkaAcl | null) => { if (acl) { modal('Are you sure want to delete this ACL record?', () => deleteResource(acl) ); } }; const columns = React.useMemo[]>( () => [ { header: 'Principal', accessorKey: 'principal', size: 257, }, { header: 'Resource', accessorKey: 'resourceType', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => ( {getValue().toLowerCase()} ), size: 145, }, { header: 'Pattern', accessorKey: 'resourceName', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue, row }) => { let chipType; if ( row.original.namePatternType === KafkaAclNamePatternType.PREFIXED ) { chipType = 'default'; } if ( row.original.namePatternType === KafkaAclNamePatternType.LITERAL ) { chipType = 'secondary'; } return ( {getValue()} {chipType ? ( {row.original.namePatternType.toLowerCase()} ) : null} ); }, size: 257, }, { header: 'Host', accessorKey: 'host', size: 257, }, { header: 'Operation', accessorKey: 'operation', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => ( {getValue().toLowerCase()} ), size: 121, }, { header: 'Permission', accessorKey: 'permission', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => ( () === KafkaAclPermissionEnum.ALLOW ? 'success' : 'danger' } > {getValue().toLowerCase()} ), size: 111, }, { id: 'delete', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ row }) => { return ( onDeleteClick(row.original)}> ); }, size: 76, }, ], [rowId] ); const onRowHover = (value: unknown) => { if (value && typeof value === 'object' && 'id' in value) { setRowId(value.id as string); } }; return ( <> setRowId('')} /> ); }; export default ACList; ================================================ FILE: kafka-ui-react-app/src/components/ACLPage/List/__test__/List.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/dom'; import userEvent from '@testing-library/user-event'; import { clusterACLPath } from 'lib/paths'; import ACList from 'components/ACLPage/List/List'; import { useAcls, useDeleteAcl } from 'lib/hooks/api/acl'; import { aclPayload } from 'lib/fixtures/acls'; jest.mock('lib/hooks/api/acl', () => ({ useAcls: jest.fn(), useDeleteAcl: jest.fn(), })); describe('ACLList Component', () => { const clusterName = 'local'; const renderComponent = () => render( , { initialEntries: [clusterACLPath(clusterName)], } ); describe('ACLList', () => { describe('when the acls are loaded', () => { beforeEach(() => { (useAcls as jest.Mock).mockImplementation(() => ({ data: aclPayload, })); (useDeleteAcl as jest.Mock).mockImplementation(() => ({ deleteResource: jest.fn(), })); }); it('renders ACLList with records', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getAllByRole('row').length).toEqual(4); }); it('shows delete icon on hover', async () => { const { container } = renderComponent(); const [trElement] = screen.getAllByRole('row'); await userEvent.hover(trElement); const deleteElement = container.querySelector('svg'); expect(deleteElement).not.toHaveStyle({ fill: 'transparent', }); }); }); describe('when it has no acls', () => { beforeEach(() => { (useAcls as jest.Mock).mockImplementation(() => ({ data: [], })); (useDeleteAcl as jest.Mock).mockImplementation(() => ({ deleteResource: jest.fn(), })); }); it('renders empty ACLList with message', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect( screen.getByRole('row', { name: 'No ACL items found' }) ).toBeInTheDocument(); }); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/App.styled.ts ================================================ import styled from 'styled-components'; export const Layout = styled.div` min-width: 1200px; @media screen and (max-width: 1023px) { min-width: initial; } `; ================================================ FILE: kafka-ui-react-app/src/components/App.tsx ================================================ import React, { Suspense, useContext } from 'react'; import { Routes, Route, Navigate } from 'react-router-dom'; import { accessErrorPage, clusterPath, errorPage, getNonExactPath, clusterNewConfigPath, } from 'lib/paths'; import PageLoader from 'components/common/PageLoader/PageLoader'; import Dashboard from 'components/Dashboard/Dashboard'; import ClusterPage from 'components/ClusterPage/ClusterPage'; import { ThemeProvider } from 'styled-components'; import { theme, darkTheme } from 'theme/theme'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { showServerError } from 'lib/errorHandling'; import { Toaster } from 'react-hot-toast'; import GlobalCSS from 'components/globalCss'; import * as S from 'components/App.styled'; import ClusterConfigForm from 'widgets/ClusterConfigForm'; import { ThemeModeContext } from 'components/contexts/ThemeModeContext'; import ConfirmationModal from './common/ConfirmationModal/ConfirmationModal'; import { ConfirmContextProvider } from './contexts/ConfirmContext'; import { GlobalSettingsProvider } from './contexts/GlobalSettingsContext'; import ErrorPage from './ErrorPage/ErrorPage'; import { UserInfoRolesAccessProvider } from './contexts/UserInfoRolesAccessContext'; import PageContainer from './PageContainer/PageContainer'; const queryClient = new QueryClient({ defaultOptions: { queries: { suspense: true, networkMode: 'offlineFirst', onError(error) { showServerError(error as Response); }, }, mutations: { onError(error) { showServerError(error as Response); }, }, }, }); const App: React.FC = () => { const { isDarkMode } = useContext(ThemeModeContext); return ( }> {['/', '/ui', '/ui/clusters'].map((path) => ( } /> ))} } /> } /> } /> } /> } /> ); }; export default App; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/Broker.tsx ================================================ import React, { Suspense } from 'react'; import PageHeading from 'components/common/PageHeading/PageHeading'; import * as Metrics from 'components/common/Metrics'; import BytesFormatted from 'components/common/BytesFormatted/BytesFormatted'; import useAppParams from 'lib/hooks/useAppParams'; import { clusterBrokerMetricsPath, clusterBrokerMetricsRelativePath, clusterBrokerConfigsPath, ClusterBrokerParam, clusterBrokerPath, clusterBrokersPath, clusterBrokerConfigsRelativePath, } from 'lib/paths'; import { useClusterStats } from 'lib/hooks/api/clusters'; import { useBrokers } from 'lib/hooks/api/brokers'; import { NavLink, Route, Routes } from 'react-router-dom'; import BrokerLogdir from 'components/Brokers/Broker/BrokerLogdir/BrokerLogdir'; import BrokerMetrics from 'components/Brokers/Broker/BrokerMetrics/BrokerMetrics'; import Navbar from 'components/common/Navigation/Navbar.styled'; import PageLoader from 'components/common/PageLoader/PageLoader'; import { ActionNavLink } from 'components/common/ActionComponent'; import { Action, ResourceType } from 'generated-sources'; import Configs from './Configs/Configs'; const Broker: React.FC = () => { const { clusterName, brokerId } = useAppParams(); const { data: clusterStats } = useClusterStats(clusterName); const { data: brokers } = useBrokers(clusterName); if (!clusterStats) return null; const brokerItem = brokers?.find(({ id }) => id === Number(brokerId)); const brokerDiskUsage = clusterStats.diskUsage?.find( (item) => item.brokerId === Number(brokerId) ); return ( <> {brokerDiskUsage?.segmentCount} {brokerItem?.port} {brokerItem?.host} (isActive ? 'is-active' : '')} end > Log directories (isActive ? 'is-active' : '')} > Configs (isActive ? 'is-active' : '')} permission={{ resource: ResourceType.CLUSTERCONFIG, action: Action.VIEW, }} > Metrics }> } /> } /> } /> ); }; export default Broker; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/BrokerLogdir.tsx ================================================ import React from 'react'; import useAppParams from 'lib/hooks/useAppParams'; import { ClusterBrokerParam } from 'lib/paths'; import { useBrokerLogDirs } from 'lib/hooks/api/brokers'; import Table from 'components/common/NewTable'; import { ColumnDef } from '@tanstack/react-table'; import { BrokersLogdirs } from 'generated-sources'; const BrokerLogdir: React.FC = () => { const { clusterName, brokerId } = useAppParams(); const { data } = useBrokerLogDirs(clusterName, Number(brokerId)); const columns = React.useMemo[]>( () => [ { header: 'Name', accessorKey: 'name' }, { header: 'Error', accessorKey: 'error' }, { header: 'Topics', accessorKey: 'topics', cell: ({ getValue }) => getValue()?.length || 0, enableSorting: false, }, { id: 'partitions', header: 'Partitions', accessorKey: 'topics', cell: ({ getValue }) => { const topics = getValue(); if (!topics) { return 0; } return topics.reduce( (acc, topic) => acc + (topic.partitions?.length || 0), 0 ); }, enableSorting: false, }, ], [] ); return (
); }; export default BrokerLogdir; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/BrokerLogdir/__test__/BrokerLogdir.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/dom'; import { clusterBrokerPath } from 'lib/paths'; import { brokerLogDirsPayload } from 'lib/fixtures/brokers'; import { useBrokerLogDirs } from 'lib/hooks/api/brokers'; import { BrokerLogdirs } from 'generated-sources'; import BrokerLogdir from 'components/Brokers/Broker/BrokerLogdir/BrokerLogdir'; jest.mock('lib/hooks/api/brokers', () => ({ useBrokerLogDirs: jest.fn(), })); const clusterName = 'local'; const brokerId = 1; describe('BrokerLogdir Component', () => { const renderComponent = async (payload?: BrokerLogdirs[]) => { (useBrokerLogDirs as jest.Mock).mockImplementation(() => ({ data: payload, })); await render( , { initialEntries: [clusterBrokerPath(clusterName, brokerId)], } ); }; it('shows warning when server returns undefined logDirs response', async () => { await renderComponent(); expect( screen.getByRole('row', { name: 'Log dir data not available' }) ).toBeInTheDocument(); }); it('shows warning when server returns empty logDirs response', async () => { await renderComponent([]); expect( screen.getByRole('row', { name: 'Log dir data not available' }) ).toBeInTheDocument(); }); it('shows brokers', async () => { await renderComponent(brokerLogDirsPayload); expect( screen.queryByRole('row', { name: 'Log dir data not available' }) ).not.toBeInTheDocument(); expect( screen.getByRole('row', { name: '/opt/kafka/data-0/logs NONE 3 4', }) ).toBeInTheDocument(); expect( screen.getByRole('row', { name: '/opt/kafka/data-1/logs NONE 0 0', }) ).toBeInTheDocument(); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/BrokerMetrics/BrokerMetrics.tsx ================================================ import React from 'react'; import useAppParams from 'lib/hooks/useAppParams'; import { ClusterBrokerParam } from 'lib/paths'; import { useBrokerMetrics } from 'lib/hooks/api/brokers'; import { SchemaType } from 'generated-sources'; import EditorViewer from 'components/common/EditorViewer/EditorViewer'; import { getEditorText } from 'components/Brokers/utils/getEditorText'; const BrokerMetrics: React.FC = () => { const { clusterName, brokerId } = useAppParams(); const { data: metrics } = useBrokerMetrics(clusterName, Number(brokerId)); return ( ); }; export default BrokerMetrics; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/BrokerMetrics/__test__/BrokerMetrics.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/dom'; import { clusterBrokerMetricsPath } from 'lib/paths'; import BrokerMetrics from 'components/Brokers/Broker/BrokerMetrics/BrokerMetrics'; import { useBrokerMetrics } from 'lib/hooks/api/brokers'; jest.mock('lib/hooks/api/brokers', () => ({ useBrokerMetrics: jest.fn(), })); const clusterName = 'local'; const brokerId = 1; describe('BrokerMetrics Component', () => { it("shows warning when server doesn't return metrics response", async () => { (useBrokerMetrics as jest.Mock).mockImplementation(() => ({ data: {}, })); render( , { initialEntries: [clusterBrokerMetricsPath(clusterName, brokerId)], } ); expect(screen.getAllByRole('textbox').length).toEqual(1); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/Configs/Configs.styled.ts ================================================ import styled from 'styled-components'; export const ValueWrapper = styled.div` display: flex; justify-content: space-between; button { margin: 0 10px; } `; export const Value = styled.span` line-height: 24px; margin-right: 10px; text-overflow: ellipsis; max-width: 400px; overflow: hidden; white-space: nowrap; `; export const ButtonsWrapper = styled.div` display: flex; `; export const SearchWrapper = styled.div` margin: 10px; width: 21%; `; export const Source = styled.div` display: flex; align-content: center; svg { margin-left: 10px; vertical-align: middle; cursor: pointer; } `; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/Configs/Configs.tsx ================================================ import React from 'react'; import { CellContext, ColumnDef } from '@tanstack/react-table'; import { ClusterBrokerParam } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import { useBrokerConfig, useUpdateBrokerConfigByName, } from 'lib/hooks/api/brokers'; import Table from 'components/common/NewTable'; import { BrokerConfig, ConfigSource } from 'generated-sources'; import Search from 'components/common/Search/Search'; import Tooltip from 'components/common/Tooltip/Tooltip'; import InfoIcon from 'components/common/Icons/InfoIcon'; import InputCell from './InputCell'; import * as S from './Configs.styled'; const tooltipContent = `DYNAMIC_TOPIC_CONFIG = dynamic topic config that is configured for a specific topic DYNAMIC_BROKER_LOGGER_CONFIG = dynamic broker logger config that is configured for a specific broker DYNAMIC_BROKER_CONFIG = dynamic broker config that is configured for a specific broker DYNAMIC_DEFAULT_BROKER_CONFIG = dynamic broker config that is configured as default for all brokers in the cluster STATIC_BROKER_CONFIG = static broker config provided as broker properties at start up (e.g. server.properties file) DEFAULT_CONFIG = built-in default configuration for configs that have a default value UNKNOWN = source unknown e.g. in the ConfigEntry used for alter requests where source is not set`; const Configs: React.FC = () => { const [keyword, setKeyword] = React.useState(''); const { clusterName, brokerId } = useAppParams(); const { data = [] } = useBrokerConfig(clusterName, Number(brokerId)); const stateMutation = useUpdateBrokerConfigByName( clusterName, Number(brokerId) ); const getData = () => { return data .filter((item) => { const nameMatch = item.name .toLocaleLowerCase() .includes(keyword.toLocaleLowerCase()); return nameMatch ? true : item.value && item.value .toLocaleLowerCase() .includes(keyword.toLocaleLowerCase()); // try to match the keyword on any of the item.value elements when nameMatch fails but item.value exists }) .sort((a, b) => { if (a.source === b.source) return 0; return a.source === ConfigSource.DYNAMIC_BROKER_CONFIG ? -1 : 1; }); }; const dataSource = React.useMemo(() => getData(), [data, keyword]); const renderCell = (props: CellContext) => ( { stateMutation.mutateAsync({ name, brokerConfigItem: { value, }, }); }} /> ); const columns = React.useMemo[]>( () => [ { header: 'Key', accessorKey: 'name' }, { header: 'Value', accessorKey: 'value', cell: renderCell, }, { // eslint-disable-next-line react/no-unstable-nested-components header: () => { return ( Source } content={tooltipContent} placement="top-end" /> ); }, accessorKey: 'source', }, ], [] ); return ( <>
); }; export default Configs; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/Configs/InputCell.tsx ================================================ import React, { useEffect } from 'react'; import { CellContext } from '@tanstack/react-table'; import CheckmarkIcon from 'components/common/Icons/CheckmarkIcon'; import EditIcon from 'components/common/Icons/EditIcon'; import CancelIcon from 'components/common/Icons/CancelIcon'; import { useConfirm } from 'lib/hooks/useConfirm'; import { Action, BrokerConfig, ResourceType } from 'generated-sources'; import { Button } from 'components/common/Button/Button'; import Input from 'components/common/Input/Input'; import { ActionButton } from 'components/common/ActionComponent'; import * as S from './Configs.styled'; interface InputCellProps extends CellContext { onUpdate: (name: string, value?: string) => void; } const InputCell: React.FC = ({ row, getValue, onUpdate }) => { const initialValue = `${getValue()}`; const [isEdit, setIsEdit] = React.useState(false); const [value, setValue] = React.useState(initialValue); const confirm = useConfirm(); const onSave = () => { if (value !== initialValue) { confirm('Are you sure you want to change the value?', async () => { onUpdate(row?.original?.name, value); }); } setIsEdit(false); }; useEffect(() => { setValue(initialValue); }, [initialValue]); return isEdit ? ( setValue(target?.value)} /> ) : ( {initialValue} setIsEdit(true)} permission={{ resource: ResourceType.CLUSTERCONFIG, action: Action.EDIT, }} > Edit ); }; export default InputCell; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/Configs/__test__/Configs.spec.tsx ================================================ import React from 'react'; import { screen } from '@testing-library/dom'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterBrokerConfigsPath } from 'lib/paths'; import { useBrokerConfig } from 'lib/hooks/api/brokers'; import { brokerConfigPayload } from 'lib/fixtures/brokers'; import Configs from 'components/Brokers/Broker/Configs/Configs'; import userEvent from '@testing-library/user-event'; const clusterName = 'Cluster_Name'; const brokerId = 'Broker_Id'; jest.mock('lib/hooks/api/brokers', () => ({ useBrokerConfig: jest.fn(), useUpdateBrokerConfigByName: jest.fn(), })); describe('Configs', () => { const renderComponent = () => { const path = clusterBrokerConfigsPath(clusterName, brokerId); return render( , { initialEntries: [path] } ); }; beforeEach(() => { (useBrokerConfig as jest.Mock).mockImplementation(() => ({ data: brokerConfigPayload, })); renderComponent(); }); it('renders configs table', async () => { expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getAllByRole('row').length).toEqual( brokerConfigPayload.length + 1 ); }); it('updates textbox value', async () => { await userEvent.click(screen.getAllByLabelText('editAction')[0]); const textbox = screen.getByLabelText('inputValue'); expect(textbox).toBeInTheDocument(); expect(textbox).toHaveValue('producer'); await userEvent.type(textbox, 'new value'); expect( screen.getByRole('button', { name: 'confirmAction' }) ).toBeInTheDocument(); expect( screen.getByRole('button', { name: 'cancelAction' }) ).toBeInTheDocument(); await userEvent.click( screen.getByRole('button', { name: 'confirmAction' }) ); expect( screen.getByText('Are you sure you want to change the value?') ).toBeInTheDocument(); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Broker/__test__/Broker.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen } from '@testing-library/dom'; import { clusterBrokerMetricsPath, clusterBrokerPath, getNonExactPath, } from 'lib/paths'; import Broker from 'components/Brokers/Broker/Broker'; import { useBrokers } from 'lib/hooks/api/brokers'; import { useClusterStats } from 'lib/hooks/api/clusters'; import { brokersPayload } from 'lib/fixtures/brokers'; import { clusterStatsPayload } from 'lib/fixtures/clusters'; const clusterName = 'local'; const brokerId = 200; const activeClassName = 'is-active'; const brokerLogdir = { pageText: 'brokerLogdir', navigationName: 'Log directories', }; const brokerMetrics = { pageText: 'brokerMetrics', navigationName: 'Metrics', }; jest.mock('components/Brokers/Broker/BrokerLogdir/BrokerLogdir', () => () => (
{brokerLogdir.pageText}
)); jest.mock('components/Brokers/Broker/BrokerMetrics/BrokerMetrics', () => () => (
{brokerMetrics.pageText}
)); jest.mock('lib/hooks/api/brokers', () => ({ useBrokers: jest.fn(), })); jest.mock('lib/hooks/api/clusters', () => ({ useClusterStats: jest.fn(), })); describe('Broker Component', () => { beforeEach(() => { (useBrokers as jest.Mock).mockImplementation(() => ({ data: brokersPayload, })); (useClusterStats as jest.Mock).mockImplementation(() => ({ data: clusterStatsPayload, })); }); const renderComponent = (path = clusterBrokerPath(clusterName, brokerId)) => render( , { initialEntries: [path], } ); it('shows broker found', async () => { await renderComponent(); const brokerInfo = brokersPayload.find((broker) => broker.id === brokerId); const brokerDiskUsage = clusterStatsPayload.diskUsage.find( (disk) => disk.brokerId === brokerId ); expect( screen.getByText(brokerDiskUsage?.segmentCount || '') ).toBeInTheDocument(); expect(screen.getByText('11.77 MB')).toBeInTheDocument(); expect(screen.getByText('Segment Count')).toBeInTheDocument(); expect( screen.getByText(brokerDiskUsage?.segmentCount || '') ).toBeInTheDocument(); expect(screen.getByText('Port')).toBeInTheDocument(); expect(screen.getByText(brokerInfo?.port || '')).toBeInTheDocument(); expect(screen.getByText('Host')).toBeInTheDocument(); expect(screen.getByText(brokerInfo?.host || '')).toBeInTheDocument(); }); it('renders Broker Logdir', async () => { await renderComponent(); const logdirLink = screen.getByRole('link', { name: brokerLogdir.navigationName, }); expect(logdirLink).toBeInTheDocument(); expect(logdirLink).toHaveClass(activeClassName); expect(screen.getByText(brokerLogdir.pageText)).toBeInTheDocument(); }); it('renders Broker Metrics', async () => { await renderComponent(clusterBrokerMetricsPath(clusterName, brokerId)); const metricsLink = screen.getByRole('link', { name: brokerMetrics.navigationName, }); expect(metricsLink).toBeInTheDocument(); expect(metricsLink).toHaveClass(activeClassName); expect(screen.getByText(brokerMetrics.pageText)).toBeInTheDocument(); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/Brokers.tsx ================================================ import React from 'react'; import { Route, Routes } from 'react-router-dom'; import { getNonExactPath, RouteParams } from 'lib/paths'; import BrokersList from 'components/Brokers/BrokersList/BrokersList'; import Broker from 'components/Brokers/Broker/Broker'; import SuspenseQueryComponent from 'components/common/SuspenseQueryComponent/SuspenseQueryComponent'; const Brokers: React.FC = () => ( } /> } /> ); export default Brokers; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.styled.ts ================================================ import styled from 'styled-components'; export const RowCell = styled.div` display: flex; width: 100%; align-items: center; svg { width: 20px; padding-left: 6px; } `; export const DangerText = styled.span` color: ${({ theme }) => theme.circularAlert.color.error}; `; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/BrokersList/BrokersList.tsx ================================================ import React from 'react'; import { ClusterName } from 'redux/interfaces'; import { useNavigate } from 'react-router-dom'; import PageHeading from 'components/common/PageHeading/PageHeading'; import * as Metrics from 'components/common/Metrics'; import useAppParams from 'lib/hooks/useAppParams'; import { useBrokers } from 'lib/hooks/api/brokers'; import { useClusterStats } from 'lib/hooks/api/clusters'; import Table, { LinkCell, SizeCell } from 'components/common/NewTable'; import CheckMarkRoundIcon from 'components/common/Icons/CheckMarkRoundIcon'; import { ColumnDef } from '@tanstack/react-table'; import { clusterBrokerPath } from 'lib/paths'; import Tooltip from 'components/common/Tooltip/Tooltip'; import ColoredCell from 'components/common/NewTable/ColoredCell'; import SkewHeader from './SkewHeader/SkewHeader'; import * as S from './BrokersList.styled'; const NA = 'N/A'; const BrokersList: React.FC = () => { const navigate = useNavigate(); const { clusterName } = useAppParams<{ clusterName: ClusterName }>(); const { data: clusterStats = {} } = useClusterStats(clusterName); const { data: brokers } = useBrokers(clusterName); const { brokerCount, activeControllers, onlinePartitionCount, offlinePartitionCount, inSyncReplicasCount, outOfSyncReplicasCount, underReplicatedPartitionCount, diskUsage, version, } = clusterStats; const rows = React.useMemo(() => { let brokersResource; if (!diskUsage || !diskUsage?.length) { brokersResource = brokers?.map((broker) => { return { brokerId: broker.id, segmentSize: NA, segmentCount: NA, }; }) || []; } else { brokersResource = diskUsage; } return brokersResource.map(({ brokerId, segmentSize, segmentCount }) => { const broker = brokers?.find(({ id }) => id === brokerId); return { brokerId, size: segmentSize || NA, count: segmentCount || NA, port: broker?.port, host: broker?.host, partitionsLeader: broker?.partitionsLeader, partitionsSkew: broker?.partitionsSkew, leadersSkew: broker?.leadersSkew, inSyncPartitions: broker?.inSyncPartitions, }; }); }, [diskUsage, brokers]); const columns = React.useMemo[]>( () => [ { header: 'Broker ID', accessorKey: 'brokerId', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => ( ()}`} to={encodeURIComponent(`${getValue()}`)} /> {getValue() === activeControllers && ( } content="Active Controller" placement="right" /> )} ), }, { header: 'Disk usage', accessorKey: 'size', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue, table, cell, column, renderValue, row }) => getValue() === NA ? ( NA ) : ( ), }, { // eslint-disable-next-line react/no-unstable-nested-components header: () => , accessorKey: 'partitionsSkew', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => { const value = getValue(); return ( = 10 && value < 20} attention={value >= 20} /> ); }, }, { header: 'Leaders', accessorKey: 'partitionsLeader' }, { header: 'Leader skew', accessorKey: 'leadersSkew', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue }) => { const value = getValue(); return ( = 10 && value < 20} attention={value >= 20} /> ); }, }, { header: 'Online partitions', accessorKey: 'inSyncPartitions', // eslint-disable-next-line react/no-unstable-nested-components cell: ({ getValue, row }) => { const value = getValue(); return ( ); }, }, { header: 'Port', accessorKey: 'port' }, { header: 'Host', accessorKey: 'host', }, ], [] ); const replicas = (inSyncReplicasCount ?? 0) + (outOfSyncReplicasCount ?? 0); const areAllInSync = inSyncReplicasCount && replicas === inSyncReplicasCount; const partitionIsOffline = offlinePartitionCount && offlinePartitionCount > 0; const isActiveControllerUnKnown = typeof activeControllers === 'undefined'; return ( <> {brokerCount} {isActiveControllerUnKnown ? ( No Active Controller ) : ( activeControllers )} {version} {partitionIsOffline ? ( {onlinePartitionCount} ) : ( onlinePartitionCount )} {` of ${ (onlinePartitionCount || 0) + (offlinePartitionCount || 0) } `} {!underReplicatedPartitionCount ? ( {underReplicatedPartitionCount} ) : ( {underReplicatedPartitionCount} )} {areAllInSync ? ( replicas ) : ( {inSyncReplicasCount} )} of {replicas} {outOfSyncReplicasCount}
navigate(clusterBrokerPath(clusterName, brokerId)) } emptyMessage="No clusters are online" /> ); }; export default BrokersList; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.styled.ts ================================================ import styled from 'styled-components'; import { MessageTooltip } from 'components/common/Tooltip/Tooltip.styled'; export const CellWrapper = styled.div` display: flex; gap: 10px; ${MessageTooltip} { max-height: unset; } `; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/BrokersList/SkewHeader/SkewHeader.tsx ================================================ import React from 'react'; import Tooltip from 'components/common/Tooltip/Tooltip'; import InfoIcon from 'components/common/Icons/InfoIcon'; import * as S from './SkewHeader.styled'; const SkewHeader: React.FC = () => ( Partitions skew } content="The divergence from the average brokers' value" /> ); export default SkewHeader; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/BrokersList/__test__/BrokersList.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { screen, waitFor } from '@testing-library/dom'; import { clusterBrokerPath, clusterBrokersPath } from 'lib/paths'; import BrokersList from 'components/Brokers/BrokersList/BrokersList'; import userEvent from '@testing-library/user-event'; import { useBrokers } from 'lib/hooks/api/brokers'; import { useClusterStats } from 'lib/hooks/api/clusters'; import { brokersPayload } from 'lib/fixtures/brokers'; import { clusterStatsPayload } from 'lib/fixtures/clusters'; const mockedUsedNavigate = jest.fn(); jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), useNavigate: () => mockedUsedNavigate, })); jest.mock('lib/hooks/api/brokers', () => ({ useBrokers: jest.fn(), })); jest.mock('lib/hooks/api/clusters', () => ({ useClusterStats: jest.fn(), })); describe('BrokersList Component', () => { const clusterName = 'local'; const testInSyncReplicasCount = 798; const testOutOfSyncReplicasCount = 1; const renderComponent = () => render( , { initialEntries: [clusterBrokersPath(clusterName)], } ); describe('BrokersList', () => { describe('when the brokers are loaded', () => { beforeEach(() => { (useBrokers as jest.Mock).mockImplementation(() => ({ data: brokersPayload, })); (useClusterStats as jest.Mock).mockImplementation(() => ({ data: clusterStatsPayload, })); }); it('renders', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getAllByRole('row').length).toEqual(3); }); it('opens broker when row clicked', async () => { renderComponent(); await userEvent.click(screen.getByRole('cell', { name: '100' })); await waitFor(() => expect(mockedUsedNavigate).toBeCalledWith( clusterBrokerPath(clusterName, '100') ) ); }); it('shows warning when offlinePartitionCount > 0', async () => { (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, offlinePartitionCount: 1345, }, })); renderComponent(); const onlineWidget = screen.getByText( clusterStatsPayload.onlinePartitionCount ); expect(onlineWidget).toBeInTheDocument(); expect(onlineWidget).toHaveStyle({ color: '#E51A1A' }); }); it('shows right count when offlinePartitionCount > 0', async () => { (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, inSyncReplicasCount: testInSyncReplicasCount, outOfSyncReplicasCount: testOutOfSyncReplicasCount, }, })); renderComponent(); const onlineWidgetDef = screen.getByText(testInSyncReplicasCount); const onlineWidget = screen.getByText( `of ${testInSyncReplicasCount + testOutOfSyncReplicasCount}` ); expect(onlineWidgetDef).toBeInTheDocument(); expect(onlineWidget).toBeInTheDocument(); }); it('shows right count when inSyncReplicasCount: undefined && outOfSyncReplicasCount: 1', async () => { (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, inSyncReplicasCount: undefined, outOfSyncReplicasCount: testOutOfSyncReplicasCount, }, })); renderComponent(); const onlineWidget = screen.getByText( `of ${testOutOfSyncReplicasCount}` ); expect(onlineWidget).toBeInTheDocument(); }); it(`shows right count when inSyncReplicasCount: ${testInSyncReplicasCount} outOfSyncReplicasCount: undefined`, async () => { (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, inSyncReplicasCount: testInSyncReplicasCount, outOfSyncReplicasCount: undefined, }, })); renderComponent(); const onlineWidgetDef = screen.getByText(testInSyncReplicasCount); const onlineWidget = screen.getByText(`of ${testInSyncReplicasCount}`); expect(onlineWidgetDef).toBeInTheDocument(); expect(onlineWidget).toBeInTheDocument(); }); }); describe('BrokersList', () => { describe('when the brokers are loaded', () => { const testActiveControllers = 0; beforeEach(() => { (useBrokers as jest.Mock).mockImplementation(() => ({ data: brokersPayload, })); (useClusterStats as jest.Mock).mockImplementation(() => ({ data: clusterStatsPayload, })); }); it(`Indicates correct active cluster`, async () => { renderComponent(); await waitFor(() => expect(screen.getByRole('tooltip')).toBeInTheDocument() ); }); it(`Correct display even if there is no active cluster: ${testActiveControllers} `, async () => { (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, activeControllers: testActiveControllers, }, })); renderComponent(); await waitFor(() => expect(screen.queryByRole('tooltip')).not.toBeInTheDocument() ); }); }); }); describe('when diskUsage is empty', () => { beforeEach(() => { (useBrokers as jest.Mock).mockImplementation(() => ({ data: brokersPayload, })); (useClusterStats as jest.Mock).mockImplementation(() => ({ data: { ...clusterStatsPayload, diskUsage: undefined }, })); }); describe('when it has no brokers', () => { beforeEach(() => { (useBrokers as jest.Mock).mockImplementation(() => ({ data: [], })); }); it('renders empty table', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect( screen.getByRole('row', { name: 'No clusters are online' }) ).toBeInTheDocument(); }); }); it('renders list of all brokers', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getAllByRole('row').length).toEqual(3); }); it('opens broker when row clicked', async () => { renderComponent(); await userEvent.click(screen.getByRole('cell', { name: '100' })); await waitFor(() => expect(mockedUsedNavigate).toBeCalledWith( clusterBrokerPath(clusterName, '100') ) ); }); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/__test__/Brokers.spec.tsx ================================================ import React from 'react'; import { render } from 'lib/testHelpers'; import { screen } from '@testing-library/react'; import { clusterBrokerPath } from 'lib/paths'; import Brokers from 'components/Brokers/Brokers'; const brokersList = 'brokersList'; const broker = 'brokers'; jest.mock('components/Brokers/BrokersList/BrokersList', () => () => (
{brokersList}
)); jest.mock('components/Brokers/Broker/Broker', () => () =>
{broker}
); describe('Brokers Component', () => { const clusterName = 'clusterName'; const brokerId = '1'; const renderComponent = (path?: string) => render(, { initialEntries: path ? [path] : undefined, }); it('renders BrokersList', () => { renderComponent(); expect(screen.getByText(brokersList)).toBeInTheDocument(); }); it('renders Broker', () => { renderComponent(clusterBrokerPath(clusterName, brokerId)); expect(screen.getByText(broker)).toBeInTheDocument(); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/utils/__test__/fixtures.ts ================================================ import { BrokerMetrics } from 'generated-sources'; export const brokerMetricsPayload: BrokerMetrics = { segmentSize: 23, segmentCount: 23, metrics: [ { name: 'TotalFetchRequestsPerSec', labels: { canonicalName: 'kafka.server:name=TotalFetchRequestsPerSec,topic=_connect_status,type=BrokerTopicMetrics', }, value: 10, }, { name: 'ZooKeeperRequestLatencyMs', value: 11, }, { name: 'RequestHandlerAvgIdlePercent', }, ], }; export const transformedBrokerMetricsPayload = '{"segmentSize":23,"segmentCount":23,"metrics":[{"name":"TotalFetchRequestsPerSec","labels":{"canonicalName":"kafka.server:name=TotalFetchRequestsPerSec,topic=_connect_status,type=BrokerTopicMetrics"},"value":10},{"name":"ZooKeeperRequestLatencyMs","value":11},{"name":"RequestHandlerAvgIdlePercent"}]}'; ================================================ FILE: kafka-ui-react-app/src/components/Brokers/utils/__test__/getEditorText.spec.tsx ================================================ import { getEditorText } from 'components/Brokers/utils/getEditorText'; import { brokerMetricsPayload, transformedBrokerMetricsPayload, } from './fixtures'; describe('Get editor text', () => { it('returns error message when broker metrics is not defined', () => { expect(getEditorText(undefined)).toEqual('Metrics data not available'); }); it('returns transformed metrics text when broker logdirs metrics', () => { expect(getEditorText(brokerMetricsPayload)).toEqual( transformedBrokerMetricsPayload ); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Brokers/utils/getEditorText.ts ================================================ import { BrokerMetrics } from 'generated-sources'; export const getEditorText = (metrics: BrokerMetrics | undefined): string => metrics ? JSON.stringify(metrics) : 'Metrics data not available'; ================================================ FILE: kafka-ui-react-app/src/components/ClusterPage/ClusterConfigPage.tsx ================================================ import React from 'react'; import { useAppConfig } from 'lib/hooks/api/appConfig'; import useAppParams from 'lib/hooks/useAppParams'; import { ClusterNameRoute } from 'lib/paths'; import ClusterConfigForm from 'widgets/ClusterConfigForm'; import { getInitialFormData } from 'widgets/ClusterConfigForm/utils/getInitialFormData'; const ClusterConfigPage: React.FC = () => { const config = useAppConfig(); const { clusterName } = useAppParams(); const currentClusterConfig = React.useMemo(() => { if (config.isSuccess && !!config.data.properties?.kafka?.clusters) { const current = config.data.properties?.kafka?.clusters?.find( ({ name }) => name === clusterName ); if (current) { return getInitialFormData(current); } } return undefined; }, [clusterName, config]); if (!currentClusterConfig) { return null; } const hasCustomConfig = Object.values(currentClusterConfig.customAuth).some( (v) => !!v ); return ( ); }; export default ClusterConfigPage; ================================================ FILE: kafka-ui-react-app/src/components/ClusterPage/ClusterPage.tsx ================================================ import React, { Suspense } from 'react'; import { Routes, Navigate, Route, Outlet } from 'react-router-dom'; import useAppParams from 'lib/hooks/useAppParams'; import { ClusterFeaturesEnum } from 'generated-sources'; import { clusterBrokerRelativePath, clusterConnectorsRelativePath, clusterConnectsRelativePath, clusterConsumerGroupsRelativePath, clusterKsqlDbRelativePath, ClusterNameRoute, clusterSchemasRelativePath, clusterTopicsRelativePath, clusterConfigRelativePath, getNonExactPath, clusterAclRelativePath, } from 'lib/paths'; import ClusterContext from 'components/contexts/ClusterContext'; import PageLoader from 'components/common/PageLoader/PageLoader'; import { useClusters } from 'lib/hooks/api/clusters'; import { GlobalSettingsContext } from 'components/contexts/GlobalSettingsContext'; const Brokers = React.lazy(() => import('components/Brokers/Brokers')); const Topics = React.lazy(() => import('components/Topics/Topics')); const Schemas = React.lazy(() => import('components/Schemas/Schemas')); const Connect = React.lazy(() => import('components/Connect/Connect')); const KsqlDb = React.lazy(() => import('components/KsqlDb/KsqlDb')); const ClusterConfigPage = React.lazy( () => import('components/ClusterPage/ClusterConfigPage') ); const ConsumerGroups = React.lazy( () => import('components/ConsumerGroups/ConsumerGroups') ); const AclPage = React.lazy(() => import('components/ACLPage/ACLPage')); const ClusterPage: React.FC = () => { const { clusterName } = useAppParams(); const appInfo = React.useContext(GlobalSettingsContext); const { data } = useClusters(); const contextValue = React.useMemo(() => { const cluster = data?.find(({ name }) => name === clusterName); const features = cluster?.features || []; return { isReadOnly: cluster?.readOnly || false, hasKafkaConnectConfigured: features.includes( ClusterFeaturesEnum.KAFKA_CONNECT ), hasSchemaRegistryConfigured: features.includes( ClusterFeaturesEnum.SCHEMA_REGISTRY ), isTopicDeletionAllowed: features.includes( ClusterFeaturesEnum.TOPIC_DELETION ), hasKsqlDbConfigured: features.includes(ClusterFeaturesEnum.KSQL_DB), hasAclViewConfigured: features.includes(ClusterFeaturesEnum.KAFKA_ACL_VIEW) || features.includes(ClusterFeaturesEnum.KAFKA_ACL_EDIT), }; }, [clusterName, data]); return ( }> }> } /> } /> } /> {contextValue.hasSchemaRegistryConfigured && ( } /> )} {contextValue.hasKafkaConnectConfigured && ( } /> )} {contextValue.hasKafkaConnectConfigured && ( } /> )} {contextValue.hasKsqlDbConfigured && ( } /> )} {contextValue.hasAclViewConfigured && ( } /> )} {appInfo.hasDynamicConfig && ( } /> )} } /> ); }; export default ClusterPage; ================================================ FILE: kafka-ui-react-app/src/components/ClusterPage/__tests__/ClusterPage.spec.tsx ================================================ import React from 'react'; import { Cluster, ClusterFeaturesEnum } from 'generated-sources'; import ClusterPageComponent from 'components/ClusterPage/ClusterPage'; import { screen, waitFor } from '@testing-library/react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterBrokersPath, clusterConnectorsPath, clusterConnectsPath, clusterConsumerGroupsPath, clusterKsqlDbPath, clusterPath, clusterSchemasPath, clusterTopicsPath, } from 'lib/paths'; import { useClusters } from 'lib/hooks/api/clusters'; import { onlineClusterPayload } from 'lib/fixtures/clusters'; const CLusterCompText = { Topics: 'Topics', Schemas: 'Schemas', Connect: 'Connect', Brokers: 'Brokers', ConsumerGroups: 'ConsumerGroups', KsqlDb: 'KsqlDb', }; jest.mock('components/Topics/Topics', () => () => (
{CLusterCompText.Topics}
)); jest.mock('components/Schemas/Schemas', () => () => (
{CLusterCompText.Schemas}
)); jest.mock('components/Connect/Connect', () => () => (
{CLusterCompText.Connect}
)); jest.mock('components/Brokers/Brokers', () => () => (
{CLusterCompText.Brokers}
)); jest.mock('components/ConsumerGroups/ConsumerGroups', () => () => (
{CLusterCompText.ConsumerGroups}
)); jest.mock('components/KsqlDb/KsqlDb', () => () => (
{CLusterCompText.KsqlDb}
)); jest.mock('lib/hooks/api/clusters', () => ({ useClusters: jest.fn(), })); describe('ClusterPage', () => { const renderComponent = async (pathname: string, payload: Cluster[] = []) => { (useClusters as jest.Mock).mockImplementation(() => ({ data: payload, })); await render( , { initialEntries: [pathname] } ); await waitFor(() => { expect(screen.queryByRole('progressbar')).not.toBeInTheDocument(); }); }; it('renders Brokers', async () => { await renderComponent(clusterBrokersPath('second')); expect(screen.getByText(CLusterCompText.Brokers)).toBeInTheDocument(); }); it('renders Topics', async () => { await renderComponent(clusterTopicsPath('second')); expect(screen.getByText(CLusterCompText.Topics)).toBeInTheDocument(); }); it('renders ConsumerGroups', async () => { await renderComponent(clusterConsumerGroupsPath('second')); expect( screen.getByText(CLusterCompText.ConsumerGroups) ).toBeInTheDocument(); }); describe('configured features', () => { const itCorrectlyHandlesConfiguredSchema = ( feature: ClusterFeaturesEnum, text: string, path: string ) => { it(`renders Schemas if ${feature} is configured`, async () => { await renderComponent(path, [ { ...onlineClusterPayload, features: [feature], }, ]); expect(screen.getByText(text)).toBeInTheDocument(); }); it(`does not render Schemas if ${feature} is not configured`, async () => { await renderComponent(path, [ { ...onlineClusterPayload, features: [] }, ]); expect(screen.queryByText(text)).not.toBeInTheDocument(); }); }; itCorrectlyHandlesConfiguredSchema( ClusterFeaturesEnum.SCHEMA_REGISTRY, CLusterCompText.Schemas, clusterSchemasPath(onlineClusterPayload.name) ); itCorrectlyHandlesConfiguredSchema( ClusterFeaturesEnum.KAFKA_CONNECT, CLusterCompText.Connect, clusterConnectsPath(onlineClusterPayload.name) ); itCorrectlyHandlesConfiguredSchema( ClusterFeaturesEnum.KAFKA_CONNECT, CLusterCompText.Connect, clusterConnectorsPath(onlineClusterPayload.name) ); itCorrectlyHandlesConfiguredSchema( ClusterFeaturesEnum.KSQL_DB, CLusterCompText.KsqlDb, clusterKsqlDbPath(onlineClusterPayload.name) ); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Connect.tsx ================================================ import React from 'react'; import { Navigate, Routes, Route } from 'react-router-dom'; import { RouteParams, clusterConnectConnectorRelativePath, clusterConnectConnectorsRelativePath, clusterConnectorNewRelativePath, getNonExactPath, clusterConnectorsPath, } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import SuspenseQueryComponent from 'components/common/SuspenseQueryComponent/SuspenseQueryComponent'; import ListPage from './List/ListPage'; import New from './New/New'; import DetailsPage from './Details/DetailsPage'; const Connect: React.FC = () => { const { clusterName } = useAppParams(); return ( } /> } /> } /> } /> } /> ); }; export default Connect; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Actions/Action.styled.ts ================================================ import styled from 'styled-components'; export const ConnectorActionsWrapperStyled = styled.div` display: flex; flex-wrap: wrap; align-items: center; gap: 8px; `; export const ButtonLabel = styled.span` margin-right: 11.5px; `; export const RestartButton = styled.div` padding: 0 12px; border: none; border-radius: 4px; display: flex; -webkit-align-items: center; background: ${({ theme }) => theme.button.primary.backgroundColor.normal}; color: ${({ theme }) => theme.button.primary.color.normal}; font-size: 14px; font-weight: 500; height: 32px; `; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Actions/Actions.tsx ================================================ import React from 'react'; import { useNavigate } from 'react-router-dom'; import { useIsMutating } from '@tanstack/react-query'; import { Action, ConnectorAction, ConnectorState, ResourceType, } from 'generated-sources'; import useAppParams from 'lib/hooks/useAppParams'; import { useConnector, useDeleteConnector, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; import { clusterConnectorsPath, RouterParamsClusterConnectConnector, } from 'lib/paths'; import { useConfirm } from 'lib/hooks/useConfirm'; import { Dropdown } from 'components/common/Dropdown'; import { ActionDropdownItem } from 'components/common/ActionComponent'; import ChevronDownIcon from 'components/common/Icons/ChevronDownIcon'; import * as S from './Action.styled'; const Actions: React.FC = () => { const navigate = useNavigate(); const routerProps = useAppParams(); const mutationsNumber = useIsMutating(); const isMutating = mutationsNumber > 0; const { data: connector } = useConnector(routerProps); const confirm = useConfirm(); const deleteConnectorMutation = useDeleteConnector(routerProps); const deleteConnectorHandler = () => confirm( <> Are you sure you want to remove {routerProps.connectorName}{' '} connector? , async () => { try { await deleteConnectorMutation.mutateAsync(); navigate(clusterConnectorsPath(routerProps.clusterName)); } catch { // do not redirect } } ); const stateMutation = useUpdateConnectorState(routerProps); const restartConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART); const restartAllTasksHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART_ALL_TASKS); const restartFailedTasksHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART_FAILED_TASKS); const pauseConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.PAUSE); const resumeConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESUME); return ( Restart } > {connector?.status.state === ConnectorState.RUNNING && ( Pause )} {connector?.status.state === ConnectorState.PAUSED && ( Resume )} Restart Connector Restart All Tasks Restart Failed Tasks Delete ); }; export default Actions; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Actions/__tests__/Actions.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorPath } from 'lib/paths'; import Actions from 'components/Connect/Details/Actions/Actions'; import { ConnectorAction, ConnectorState } from 'generated-sources'; import { screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { useConnector, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; import { connector } from 'lib/fixtures/kafkaConnect'; import set from 'lodash/set'; const mockHistoryPush = jest.fn(); const deleteConnector = jest.fn(); const cancelMock = jest.fn(); jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), useNavigate: () => mockHistoryPush, })); jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnector: jest.fn(), useDeleteConnector: jest.fn(), useUpdateConnectorState: jest.fn(), })); const expectActionButtonsExists = () => { expect(screen.getByText('Restart Connector')).toBeInTheDocument(); expect(screen.getByText('Restart All Tasks')).toBeInTheDocument(); expect(screen.getByText('Restart Failed Tasks')).toBeInTheDocument(); expect(screen.getByText('Delete')).toBeInTheDocument(); }; const afterClickDropDownButton = async () => { const dropDownButton = screen.getAllByRole('button'); await userEvent.click(dropDownButton[1]); }; const afterClickRestartButton = async () => { const dropDownButton = screen.getByText('Restart'); await userEvent.click(dropDownButton); }; describe('Actions', () => { afterEach(() => { mockHistoryPush.mockClear(); deleteConnector.mockClear(); cancelMock.mockClear(); }); describe('view', () => { const route = clusterConnectConnectorPath(); const path = clusterConnectConnectorPath( 'myCluster', 'myConnect', 'myConnector' ); const renderComponent = () => render( , { initialEntries: [path] } ); it('renders buttons when paused', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.PAUSED), })); renderComponent(); await afterClickRestartButton(); expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.getByText('Resume')).toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); it('renders buttons when failed', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.FAILED), })); renderComponent(); await afterClickRestartButton(); expect(screen.getAllByRole('menuitem').length).toEqual(3); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); it('renders buttons when unassigned', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.UNASSIGNED), })); renderComponent(); await afterClickRestartButton(); expect(screen.getAllByRole('menuitem').length).toEqual(3); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.queryByText('Pause')).not.toBeInTheDocument(); expectActionButtonsExists(); }); it('renders buttons when running connector action', async () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.RUNNING), })); renderComponent(); await afterClickRestartButton(); expect(screen.getAllByRole('menuitem').length).toEqual(4); expect(screen.queryByText('Resume')).not.toBeInTheDocument(); expect(screen.getByText('Pause')).toBeInTheDocument(); expectActionButtonsExists(); }); describe('mutations', () => { beforeEach(() => { (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.RUNNING), })); }); it('opens confirmation modal when delete button clicked', async () => { renderComponent(); await afterClickDropDownButton(); await waitFor(async () => userEvent.click(screen.getByRole('menuitem', { name: 'Delete' })) ); expect(screen.getByRole('dialog')).toBeInTheDocument(); }); it('calls restartConnector when restart button clicked', async () => { const restartConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartConnector, })); renderComponent(); await afterClickRestartButton(); await userEvent.click( screen.getByRole('menuitem', { name: 'Restart Connector' }) ); expect(restartConnector).toHaveBeenCalledWith(ConnectorAction.RESTART); }); it('calls restartAllTasks', async () => { const restartAllTasks = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartAllTasks, })); renderComponent(); await afterClickRestartButton(); await userEvent.click( screen.getByRole('menuitem', { name: 'Restart All Tasks' }) ); expect(restartAllTasks).toHaveBeenCalledWith( ConnectorAction.RESTART_ALL_TASKS ); }); it('calls restartFailedTasks', async () => { const restartFailedTasks = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartFailedTasks, })); renderComponent(); await afterClickRestartButton(); await userEvent.click( screen.getByRole('menuitem', { name: 'Restart Failed Tasks' }) ); expect(restartFailedTasks).toHaveBeenCalledWith( ConnectorAction.RESTART_FAILED_TASKS ); }); it('calls pauseConnector when pause button clicked', async () => { const pauseConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: pauseConnector, })); renderComponent(); await afterClickRestartButton(); await userEvent.click(screen.getByRole('menuitem', { name: 'Pause' })); expect(pauseConnector).toHaveBeenCalledWith(ConnectorAction.PAUSE); }); it('calls resumeConnector when resume button clicked', async () => { const resumeConnector = jest.fn(); (useConnector as jest.Mock).mockImplementation(() => ({ data: set({ ...connector }, 'status.state', ConnectorState.PAUSED), })); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: resumeConnector, })); renderComponent(); await afterClickRestartButton(); await userEvent.click(screen.getByRole('menuitem', { name: 'Resume' })); expect(resumeConnector).toHaveBeenCalledWith(ConnectorAction.RESUME); }); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Config/Config.styled.ts ================================================ import styled from 'styled-components'; export const ConnectEditWrapperStyled = styled.div` margin: 16px; & form > *:last-child { margin-top: 16px; } `; export const ConnectEditWarningMessageStyled = styled.div` height: 48px; display: flex; align-items: center; background-color: ${({ theme }) => theme.connectEditWarning}; border-radius: 8px; padding: 8px; margin-bottom: 16px; `; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Config/Config.tsx ================================================ import React from 'react'; import useAppParams from 'lib/hooks/useAppParams'; import { Controller, useForm } from 'react-hook-form'; import { ErrorMessage } from '@hookform/error-message'; import { yupResolver } from '@hookform/resolvers/yup'; import { RouterParamsClusterConnectConnector } from 'lib/paths'; import yup from 'lib/yupExtended'; import Editor from 'components/common/Editor/Editor'; import { Button } from 'components/common/Button/Button'; import { useConnectorConfig, useUpdateConnectorConfig, } from 'lib/hooks/api/kafkaConnect'; import { ConnectEditWarningMessageStyled, ConnectEditWrapperStyled, } from './Config.styled'; const validationSchema = yup.object().shape({ config: yup.string().required().isJsonObject(), }); interface FormValues { config: string; } const Config: React.FC = () => { const routerParams = useAppParams(); const { data: config } = useConnectorConfig(routerParams); const mutation = useUpdateConnectorConfig(routerParams); const { handleSubmit, control, reset, formState: { isDirty, isSubmitting, isValid, errors }, setValue, } = useForm({ mode: 'onChange', resolver: yupResolver(validationSchema), defaultValues: { config: JSON.stringify(config, null, '\t'), }, }); React.useEffect(() => { if (config) { setValue('config', JSON.stringify(config, null, '\t')); } }, [config, setValue]); const onSubmit = async (values: FormValues) => { try { const requestBody = JSON.parse(values.config.trim()); await mutation.mutateAsync(requestBody); reset(values); } catch (e) { // do nothing } }; const hasCredentials = JSON.stringify(config, null, '\t').includes( '"******"' ); return ( {hasCredentials && ( Please replace ****** with the real credential values to avoid accidentally breaking your connector config! )}
( )} />
); }; export default Config; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Config/__tests__/Config.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorConfigPath } from 'lib/paths'; import Config from 'components/Connect/Details/Config/Config'; import { connector } from 'lib/fixtures/kafkaConnect'; import { waitFor } from '@testing-library/dom'; import { act, fireEvent, screen } from '@testing-library/react'; import { useConnectorConfig, useUpdateConnectorConfig, } from 'lib/hooks/api/kafkaConnect'; jest.mock('components/common/Editor/Editor', () => 'mock-Editor'); const mockHistoryPush = jest.fn(); jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), useNavigate: () => mockHistoryPush, })); jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectorConfig: jest.fn(), useUpdateConnectorConfig: jest.fn(), })); const [clusterName, connectName, connectorName] = [ 'my-cluster', 'my-connect', 'my-connector', ]; describe('Config', () => { const pathname = clusterConnectConnectorConfigPath(); const renderComponent = () => render( , { initialEntries: [ clusterConnectConnectorConfigPath( clusterName, connectName, connectorName ), ], } ); beforeEach(() => { (useConnectorConfig as jest.Mock).mockImplementation(() => ({ data: connector.config, })); }); it('calls updateConfig and redirects to connector config view on successful submit', async () => { const updateConfig = jest.fn(() => { return Promise.resolve(connector); }); (useUpdateConnectorConfig as jest.Mock).mockImplementation(() => ({ mutateAsync: updateConfig, })); renderComponent(); fireEvent.submit(screen.getByRole('form')); await waitFor(() => expect(updateConfig).toHaveBeenCalledTimes(1)); }); it('does not redirect to connector config view on unsuccessful submit', async () => { const updateConfig = jest.fn(() => { return Promise.resolve(); }); (useUpdateConnectorConfig as jest.Mock).mockImplementation(() => ({ mutateAsync: updateConfig, })); renderComponent(); await act(() => { fireEvent.submit(screen.getByRole('form')); }); expect(mockHistoryPush).not.toHaveBeenCalled(); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/DetailsPage.tsx ================================================ import React, { Suspense } from 'react'; import { NavLink, Route, Routes } from 'react-router-dom'; import useAppParams from 'lib/hooks/useAppParams'; import { clusterConnectConnectorConfigPath, clusterConnectConnectorConfigRelativePath, clusterConnectConnectorPath, clusterConnectorsPath, RouterParamsClusterConnectConnector, } from 'lib/paths'; import Navbar from 'components/common/Navigation/Navbar.styled'; import PageHeading from 'components/common/PageHeading/PageHeading'; import PageLoader from 'components/common/PageLoader/PageLoader'; import Overview from './Overview/Overview'; import Tasks from './Tasks/Tasks'; import Config from './Config/Config'; import Actions from './Actions/Actions'; const DetailsPage: React.FC = () => { const { clusterName, connectName, connectorName } = useAppParams(); return (
(isActive ? 'is-active' : '')} end > Tasks (isActive ? 'is-active' : '')} > Config }> } /> } />
); }; export default DetailsPage; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Overview/Overview.tsx ================================================ import React from 'react'; import * as C from 'components/common/Tag/Tag.styled'; import * as Metrics from 'components/common/Metrics'; import getTagColor from 'components/common/Tag/getTagColor'; import { RouterParamsClusterConnectConnector } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import { useConnector, useConnectorTasks } from 'lib/hooks/api/kafkaConnect'; import getTaskMetrics from './getTaskMetrics'; const Overview: React.FC = () => { const routerProps = useAppParams(); const { data: connector } = useConnector(routerProps); const { data: tasks } = useConnectorTasks(routerProps); if (!connector) { return null; } const { running, failed } = getTaskMetrics(tasks); return ( {connector.status?.workerId && ( {connector.status.workerId} )} {connector.type} {connector.config['connector.class'] && ( {connector.config['connector.class']} )} {connector.status.state} {running} 0 ? 'error' : 'success'} > {failed} ); }; export default Overview; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Overview/__tests__/Overview.spec.tsx ================================================ import React from 'react'; import Overview from 'components/Connect/Details/Overview/Overview'; import { connector, tasks } from 'lib/fixtures/kafkaConnect'; import { screen } from '@testing-library/react'; import { render } from 'lib/testHelpers'; import { useConnector, useConnectorTasks } from 'lib/hooks/api/kafkaConnect'; jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnector: jest.fn(), useConnectorTasks: jest.fn(), })); describe('Overview', () => { it('is empty when no connector', () => { (useConnector as jest.Mock).mockImplementation(() => ({ data: undefined, })); (useConnectorTasks as jest.Mock).mockImplementation(() => ({ data: undefined, })); render(); expect(screen.queryByText('Worker')).not.toBeInTheDocument(); }); describe('when connector is loaded', () => { beforeEach(() => { (useConnector as jest.Mock).mockImplementation(() => ({ data: connector, })); }); beforeEach(() => { (useConnectorTasks as jest.Mock).mockImplementation(() => ({ data: tasks, })); }); it('renders metrics', () => { render(); expect(screen.getByText('Worker')).toBeInTheDocument(); expect( screen.getByText(connector.status.workerId as string) ).toBeInTheDocument(); expect(screen.getByText('Type')).toBeInTheDocument(); expect( screen.getByText(connector.config['connector.class'] as string) ).toBeInTheDocument(); expect(screen.getByText('Tasks Running')).toBeInTheDocument(); expect(screen.getByText(2)).toBeInTheDocument(); expect(screen.getByText('Tasks Failed')).toBeInTheDocument(); expect(screen.getByText(1)).toBeInTheDocument(); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Overview/__tests__/getTaskMetrics.spec.ts ================================================ import { tasks } from 'lib/fixtures/kafkaConnect'; import getTaskMetrics from 'components/Connect/Details/Overview/getTaskMetrics'; describe('getTaskMetrics', () => { it('should return the correct metrics when task list is undefined', () => { const metrics = getTaskMetrics(); expect(metrics).toEqual({ running: 0, failed: 0, }); }); it('should return the correct metrics', () => { expect(getTaskMetrics(tasks)).toEqual({ running: 2, failed: 1, }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Overview/getTaskMetrics.ts ================================================ import { ConnectorTaskStatus, Task } from 'generated-sources'; export default function getTaskMetrics(tasks?: Task[]) { const initialMetrics = { running: 0, failed: 0, }; if (!tasks) { return initialMetrics; } return tasks.reduce((acc, { status }) => { const state = status?.state; if (state === ConnectorTaskStatus.RUNNING) { return { ...acc, running: acc.running + 1 }; } if (state === ConnectorTaskStatus.FAILED) { return { ...acc, failed: acc.failed + 1 }; } return acc; }, initialMetrics); } ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Tasks/ActionsCellTasks.tsx ================================================ import React from 'react'; import { Action, ResourceType, Task } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; import useAppParams from 'lib/hooks/useAppParams'; import { useRestartConnectorTask } from 'lib/hooks/api/kafkaConnect'; import { Dropdown } from 'components/common/Dropdown'; import { ActionDropdownItem } from 'components/common/ActionComponent'; import { RouterParamsClusterConnectConnector } from 'lib/paths'; const ActionsCellTasks: React.FC> = ({ row }) => { const { id } = row.original; const routerProps = useAppParams(); const restartMutation = useRestartConnectorTask(routerProps); const restartTaskHandler = (taskId?: number) => { if (taskId === undefined) return; restartMutation.mutateAsync(taskId); }; return ( restartTaskHandler(id?.task)} danger confirm="Are you sure you want to restart the task?" permission={{ resource: ResourceType.CONNECT, action: Action.RESTART, value: routerProps.connectorName, }} > Restart task ); }; export default ActionsCellTasks; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Tasks/Tasks.tsx ================================================ import React from 'react'; import { useConnectorTasks } from 'lib/hooks/api/kafkaConnect'; import useAppParams from 'lib/hooks/useAppParams'; import { RouterParamsClusterConnectConnector } from 'lib/paths'; import { ColumnDef, Row } from '@tanstack/react-table'; import { Task } from 'generated-sources'; import Table, { TagCell } from 'components/common/NewTable'; import ActionsCellTasks from './ActionsCellTasks'; const ExpandedTaskRow: React.FC<{ row: Row }> = ({ row }) => { return
{row.original.status.trace}
; }; const MAX_LENGTH = 100; const Tasks: React.FC = () => { const routerProps = useAppParams(); const { data = [] } = useConnectorTasks(routerProps); const columns = React.useMemo[]>( () => [ { header: 'ID', accessorKey: 'status.id' }, { header: 'Worker', accessorKey: 'status.workerId' }, { header: 'State', accessorKey: 'status.state', cell: TagCell }, { header: 'Trace', accessorKey: 'status.trace', enableSorting: false, cell: ({ getValue }) => { const trace = getValue() || ''; return trace.toString().length > MAX_LENGTH ? `${trace.toString().substring(0, MAX_LENGTH - 3)}...` : trace; }, meta: { width: '70%' }, }, { id: 'actions', header: '', cell: ActionsCellTasks, }, ], [] ); return (
row.original.status.trace?.length > 0} renderSubComponent={ExpandedTaskRow} /> ); }; export default Tasks; ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/Tasks/__tests__/Tasks.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorTasksPath } from 'lib/paths'; import Tasks from 'components/Connect/Details/Tasks/Tasks'; import { tasks } from 'lib/fixtures/kafkaConnect'; import { screen, within, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { useConnectorTasks, useRestartConnectorTask, } from 'lib/hooks/api/kafkaConnect'; import { Task } from 'generated-sources'; jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectorTasks: jest.fn(), useRestartConnectorTask: jest.fn(), })); const path = clusterConnectConnectorTasksPath('local', 'ghp', '1'); const restartConnectorMock = jest.fn(); describe('Tasks', () => { beforeEach(() => { (useRestartConnectorTask as jest.Mock).mockImplementation(() => ({ mutateAsync: restartConnectorMock, })); }); const renderComponent = (currentData: Task[] | undefined = undefined) => { (useConnectorTasks as jest.Mock).mockImplementation(() => ({ data: currentData, })); render( , { initialEntries: [path] } ); }; it('renders empty table', () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getByText('No tasks found')).toBeInTheDocument(); }); it('renders tasks table', () => { renderComponent(tasks); expect(screen.getAllByRole('row').length).toEqual(tasks.length + 1); expect( screen.getByRole('row', { name: '1 kafka-connect0:8083 RUNNING', }) ).toBeInTheDocument(); }); it('renders truncates long trace and expands', async () => { renderComponent(tasks); const trace = tasks[2]?.status?.trace || ''; const truncatedTrace = trace.toString().substring(0, 100 - 3); const thirdRow = screen.getByRole('row', { name: `3 kafka-connect0:8083 RUNNING ${truncatedTrace}...`, }); expect(thirdRow).toBeInTheDocument(); const expandedDetails = screen.queryByText(trace); // Full trace is not visible expect(expandedDetails).not.toBeInTheDocument(); await userEvent.click(thirdRow); expect( screen.getByRole('row', { name: trace, }) ).toBeInTheDocument(); }); describe('Action button', () => { const expectDropdownExists = async () => { const firstTaskRow = screen.getByRole('row', { name: '1 kafka-connect0:8083 RUNNING', }); expect(firstTaskRow).toBeInTheDocument(); const extBtn = within(firstTaskRow).getByRole('button', { name: 'Dropdown Toggle', }); expect(extBtn).toBeEnabled(); await userEvent.click(extBtn); expect(screen.getByRole('menu')).toBeInTheDocument(); }; it('renders action button', async () => { renderComponent(tasks); await expectDropdownExists(); expect( screen.getAllByRole('button', { name: 'Dropdown Toggle' }).length ).toEqual(tasks.length); // Action buttons are enabled const actionBtn = screen.getAllByRole('menuitem'); expect(actionBtn[0]).toHaveTextContent('Restart task'); }); it('works as expected', async () => { renderComponent(tasks); await expectDropdownExists(); const actionBtn = screen.getAllByRole('menuitem'); expect(actionBtn[0]).toHaveTextContent('Restart task'); await userEvent.click(actionBtn[0]); expect( screen.getByText('Are you sure you want to restart the task?') ).toBeInTheDocument(); expect(screen.getByText('Confirm the action')).toBeInTheDocument(); userEvent.click(screen.getByRole('button', { name: 'Confirm' })); await waitFor(() => expect(restartConnectorMock).toHaveBeenCalled()); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/Details/__tests__/DetailsPage.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorConfigPath, clusterConnectConnectorPath, getNonExactPath, } from 'lib/paths'; import { screen } from '@testing-library/dom'; import DetailsPage from 'components/Connect/Details/DetailsPage'; const DetailsCompText = { overview: 'Overview Pane', tasks: 'Tasks Page', config: 'Config Page', actions: 'Actions', }; jest.mock('components/Connect/Details/Overview/Overview', () => () => (
{DetailsCompText.overview}
)); jest.mock('components/Connect/Details/Tasks/Tasks', () => () => (
{DetailsCompText.tasks}
)); jest.mock('components/Connect/Details/Config/Config', () => () => (
{DetailsCompText.config}
)); jest.mock('components/Connect/Details/Actions/Actions', () => () => (
{DetailsCompText.actions}
)); describe('Details Page', () => { const clusterName = 'my-cluster'; const connectName = 'my-connect'; const connectorName = 'my-connector'; const defaultPath = clusterConnectConnectorPath( clusterName, connectName, connectorName ); const renderComponent = (path: string = defaultPath) => render( , { initialEntries: [path] } ); it('renders actions', () => { renderComponent(); expect(screen.getByText(DetailsCompText.actions)); }); it('renders overview pane', () => { renderComponent(); expect(screen.getByText(DetailsCompText.overview)); }); describe('Router component tests', () => { it('should test if tasks is rendering', () => { renderComponent(); expect(screen.getByText(DetailsCompText.tasks)); }); it('should test if list is rendering', () => { const path = clusterConnectConnectorConfigPath( clusterName, connectName, connectorName ); renderComponent(path); expect(screen.getByText(DetailsCompText.config)); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/ActionsCell.tsx ================================================ import React from 'react'; import { Action, ConnectorAction, ConnectorState, FullConnectorInfo, ResourceType, } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; import { ClusterNameRoute } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import { Dropdown, DropdownItem } from 'components/common/Dropdown'; import { useDeleteConnector, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; import { useConfirm } from 'lib/hooks/useConfirm'; import { useIsMutating } from '@tanstack/react-query'; import { ActionDropdownItem } from 'components/common/ActionComponent'; const ActionsCell: React.FC> = ({ row, }) => { const { connect, name, status } = row.original; const { clusterName } = useAppParams(); const mutationsNumber = useIsMutating(); const isMutating = mutationsNumber > 0; const confirm = useConfirm(); const deleteMutation = useDeleteConnector({ clusterName, connectName: connect, connectorName: name, }); const stateMutation = useUpdateConnectorState({ clusterName, connectName: connect, connectorName: name, }); const handleDelete = () => { confirm( <> Are you sure want to remove {name} connector? , async () => { await deleteMutation.mutateAsync(); } ); }; // const stateMutation = useUpdateConnectorState(routerProps); const resumeConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESUME); const restartConnectorHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART); const restartAllTasksHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART_ALL_TASKS); const restartFailedTasksHandler = () => stateMutation.mutateAsync(ConnectorAction.RESTART_FAILED_TASKS); return ( {status.state === ConnectorState.PAUSED && ( Resume )} Restart Connector Restart All Tasks Restart Failed Tasks Remove Connector ); }; export default ActionsCell; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/List.styled.ts ================================================ import styled from 'styled-components'; export const TagsWrapper = styled.div` display: flex; flex-wrap: wrap; span { color: rgb(76, 76, 255) !important; &:hover { color: rgb(23, 23, 207) !important; } } `; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/List.tsx ================================================ import React from 'react'; import useAppParams from 'lib/hooks/useAppParams'; import { clusterConnectConnectorPath, ClusterNameRoute } from 'lib/paths'; import Table, { TagCell } from 'components/common/NewTable'; import { FullConnectorInfo } from 'generated-sources'; import { useConnectors } from 'lib/hooks/api/kafkaConnect'; import { ColumnDef } from '@tanstack/react-table'; import { useNavigate, useSearchParams } from 'react-router-dom'; import ActionsCell from './ActionsCell'; import TopicsCell from './TopicsCell'; import RunningTasksCell from './RunningTasksCell'; const List: React.FC = () => { const navigate = useNavigate(); const { clusterName } = useAppParams(); const [searchParams] = useSearchParams(); const { data: connectors } = useConnectors( clusterName, searchParams.get('q') || '' ); const columns = React.useMemo[]>( () => [ { header: 'Name', accessorKey: 'name' }, { header: 'Connect', accessorKey: 'connect' }, { header: 'Type', accessorKey: 'type' }, { header: 'Plugin', accessorKey: 'connectorClass' }, { header: 'Topics', cell: TopicsCell }, { header: 'Status', accessorKey: 'status.state', cell: TagCell }, { header: 'Running Tasks', cell: RunningTasksCell }, { header: '', id: 'action', cell: ActionsCell }, ], [] ); return (
navigate(clusterConnectConnectorPath(clusterName, connect, name)) } emptyMessage="No connectors found" /> ); }; export default List; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/ListPage.tsx ================================================ import React, { Suspense } from 'react'; import useAppParams from 'lib/hooks/useAppParams'; import { clusterConnectorNewRelativePath, ClusterNameRoute } from 'lib/paths'; import ClusterContext from 'components/contexts/ClusterContext'; import Search from 'components/common/Search/Search'; import * as Metrics from 'components/common/Metrics'; import PageHeading from 'components/common/PageHeading/PageHeading'; import { ActionButton } from 'components/common/ActionComponent'; import { ControlPanelWrapper } from 'components/common/ControlPanel/ControlPanel.styled'; import PageLoader from 'components/common/PageLoader/PageLoader'; import { Action, ConnectorState, ResourceType } from 'generated-sources'; import { useConnectors } from 'lib/hooks/api/kafkaConnect'; import List from './List'; const ListPage: React.FC = () => { const { isReadOnly } = React.useContext(ClusterContext); const { clusterName } = useAppParams(); // Fetches all connectors from the API, without search criteria. Used to display general metrics. const { data: connectorsMetrics, isLoading } = useConnectors(clusterName); const numberOfFailedConnectors = connectorsMetrics?.filter( ({ status: { state } }) => state === ConnectorState.FAILED ).length; const numberOfFailedTasks = connectorsMetrics?.reduce( (acc, metric) => acc + (metric.failedTasksCount ?? 0), 0 ); return ( <> {!isReadOnly && ( Create Connector )} {connectorsMetrics?.length || '-'} {numberOfFailedConnectors ?? '-'} {numberOfFailedTasks ?? '-'} }> ); }; export default ListPage; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/RunningTasksCell.tsx ================================================ import React from 'react'; import { FullConnectorInfo } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; const RunningTasksCell: React.FC> = ({ row, }) => { const { tasksCount, failedTasksCount } = row.original; if (!tasksCount) { return null; } return ( <> {tasksCount - (failedTasksCount || 0)} of {tasksCount} ); }; export default RunningTasksCell; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/TopicsCell.tsx ================================================ import React from 'react'; import { FullConnectorInfo } from 'generated-sources'; import { CellContext } from '@tanstack/react-table'; import { useNavigate } from 'react-router-dom'; import { Tag } from 'components/common/Tag/Tag.styled'; import { ClusterNameRoute, clusterTopicPath } from 'lib/paths'; import useAppParams from 'lib/hooks/useAppParams'; import * as S from './List.styled'; const TopicsCell: React.FC> = ({ row, }) => { const { topics } = row.original; const { clusterName } = useAppParams(); const navigate = useNavigate(); const navigateToTopic = ( e: React.KeyboardEvent | React.MouseEvent, topic: string ) => { e.preventDefault(); e.stopPropagation(); navigate(clusterTopicPath(clusterName, topic)); }; return ( {topics?.map((t) => ( navigateToTopic(e, t)} onKeyDown={(e) => navigateToTopic(e, t)} tabIndex={0} > {t} ))} ); }; export default TopicsCell; ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/__tests__/List.spec.tsx ================================================ import React from 'react'; import { connectors } from 'lib/fixtures/kafkaConnect'; import ClusterContext, { ContextProps, initialValue, } from 'components/contexts/ClusterContext'; import List from 'components/Connect/List/List'; import { screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorPath, clusterConnectorsPath } from 'lib/paths'; import { useConnectors, useDeleteConnector, useUpdateConnectorState, } from 'lib/hooks/api/kafkaConnect'; const mockedUsedNavigate = jest.fn(); const mockDelete = jest.fn(); jest.mock('react-router-dom', () => ({ ...jest.requireActual('react-router-dom'), useNavigate: () => mockedUsedNavigate, })); jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectors: jest.fn(), useDeleteConnector: jest.fn(), useUpdateConnectorState: jest.fn(), })); const clusterName = 'local'; const renderComponent = (contextValue: ContextProps = initialValue) => render( , { initialEntries: [clusterConnectorsPath(clusterName)] } ); describe('Connectors List', () => { describe('when the connectors are loaded', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ data: connectors, })); const restartConnector = jest.fn(); (useUpdateConnectorState as jest.Mock).mockImplementation(() => ({ mutateAsync: restartConnector, })); }); it('renders', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect(screen.getAllByRole('row').length).toEqual(3); }); it('opens broker when row clicked', async () => { renderComponent(); await userEvent.click( screen.getByRole('row', { name: 'hdfs-source-connector first SOURCE FileStreamSource a b c RUNNING 2 of 2', }) ); await waitFor(() => expect(mockedUsedNavigate).toBeCalledWith( clusterConnectConnectorPath( clusterName, 'first', 'hdfs-source-connector' ) ) ); }); }); describe('when table is empty', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ data: [], })); }); it('renders empty table', async () => { renderComponent(); expect(screen.getByRole('table')).toBeInTheDocument(); expect( screen.getByRole('row', { name: 'No connectors found' }) ).toBeInTheDocument(); }); }); describe('when remove connector modal is open', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ data: connectors, })); (useDeleteConnector as jest.Mock).mockImplementation(() => ({ mutateAsync: mockDelete, })); }); it('calls removeConnector on confirm', async () => { renderComponent(); const removeButton = screen.getAllByText('Remove Connector')[0]; await waitFor(() => userEvent.click(removeButton)); const submitButton = screen.getAllByRole('button', { name: 'Confirm', })[0]; await userEvent.click(submitButton); expect(mockDelete).toHaveBeenCalledWith(); }); it('closes the modal when cancel button is clicked', async () => { renderComponent(); const removeButton = screen.getAllByText('Remove Connector')[0]; await waitFor(() => userEvent.click(removeButton)); const cancelButton = screen.getAllByRole('button', { name: 'Cancel', })[0]; await waitFor(() => userEvent.click(cancelButton)); expect(cancelButton).not.toBeInTheDocument(); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/List/__tests__/ListPage.spec.tsx ================================================ import React from 'react'; import { connectors } from 'lib/fixtures/kafkaConnect'; import ClusterContext, { ContextProps, initialValue, } from 'components/contexts/ClusterContext'; import ListPage from 'components/Connect/List/ListPage'; import { screen, within } from '@testing-library/react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectorsPath } from 'lib/paths'; import { useConnectors } from 'lib/hooks/api/kafkaConnect'; jest.mock('components/Connect/List/List', () => () => (
Connectors List
)); jest.mock('lib/hooks/api/kafkaConnect', () => ({ useConnectors: jest.fn(), })); jest.mock('components/common/Icons/SpinnerIcon', () => () => 'progressbar'); const clusterName = 'local'; describe('Connectors List Page', () => { beforeEach(() => { (useConnectors as jest.Mock).mockImplementation(() => ({ isLoading: false, data: [], })); }); const renderComponent = async (contextValue: ContextProps = initialValue) => render( , { initialEntries: [clusterConnectorsPath(clusterName)] } ); describe('Heading', () => { it('renders header without create button for readonly cluster', async () => { await renderComponent({ ...initialValue, isReadOnly: true }); expect( screen.getByRole('heading', { name: 'Connectors' }) ).toBeInTheDocument(); expect( screen.queryByRole('link', { name: 'Create Connector' }) ).not.toBeInTheDocument(); }); it('renders header with create button for read/write cluster', async () => { await renderComponent(); expect( screen.getByRole('heading', { name: 'Connectors' }) ).toBeInTheDocument(); expect( screen.getByRole('link', { name: 'Create Connector' }) ).toBeInTheDocument(); }); }); it('renders search input', async () => { await renderComponent(); expect( screen.getByPlaceholderText('Search by Connect Name, Status or Type') ).toBeInTheDocument(); }); it('renders list', async () => { await renderComponent(); expect(screen.getByText('Connectors List')).toBeInTheDocument(); }); describe('Metrics', () => { it('renders indicators in loading state', async () => { (useConnectors as jest.Mock).mockImplementation(() => ({ isLoading: true, data: connectors, })); await renderComponent(); const metrics = screen.getByRole('group'); expect(metrics).toBeInTheDocument(); expect(within(metrics).getAllByText('progressbar').length).toEqual(3); }); it('renders indicators for empty list of connectors', async () => { await renderComponent(); const metrics = screen.getByRole('group'); expect(metrics).toBeInTheDocument(); const connectorsIndicator = within(metrics).getByTitle( 'Total number of connectors' ); expect(connectorsIndicator).toBeInTheDocument(); expect(connectorsIndicator).toHaveTextContent('Connectors -'); const failedConnectorsIndicator = within(metrics).getByTitle( 'Number of failed connectors' ); expect(failedConnectorsIndicator).toBeInTheDocument(); expect(failedConnectorsIndicator).toHaveTextContent( 'Failed Connectors 0' ); const failedTasksIndicator = within(metrics).getByTitle( 'Number of failed tasks' ); expect(failedTasksIndicator).toBeInTheDocument(); expect(failedTasksIndicator).toHaveTextContent('Failed Tasks 0'); }); it('renders indicators when connectors list is undefined', async () => { (useConnectors as jest.Mock).mockImplementation(() => ({ isFetching: false, data: undefined, })); await renderComponent(); const metrics = screen.getByRole('group'); expect(metrics).toBeInTheDocument(); const connectorsIndicator = within(metrics).getByTitle( 'Total number of connectors' ); expect(connectorsIndicator).toBeInTheDocument(); expect(connectorsIndicator).toHaveTextContent('Connectors -'); const failedConnectorsIndicator = within(metrics).getByTitle( 'Number of failed connectors' ); expect(failedConnectorsIndicator).toBeInTheDocument(); expect(failedConnectorsIndicator).toHaveTextContent( 'Failed Connectors -' ); const failedTasksIndicator = within(metrics).getByTitle( 'Number of failed tasks' ); expect(failedTasksIndicator).toBeInTheDocument(); expect(failedTasksIndicator).toHaveTextContent('Failed Tasks -'); }); it('renders indicators list of connectors', async () => { (useConnectors as jest.Mock).mockImplementation(() => ({ isLoading: false, data: connectors, })); await renderComponent(); const metrics = screen.getByRole('group'); expect(metrics).toBeInTheDocument(); const connectorsIndicator = within(metrics).getByTitle( 'Total number of connectors' ); expect(connectorsIndicator).toBeInTheDocument(); expect(connectorsIndicator).toHaveTextContent( `Connectors ${connectors.length}` ); const failedConnectorsIndicator = within(metrics).getByTitle( 'Number of failed connectors' ); expect(failedConnectorsIndicator).toBeInTheDocument(); expect(failedConnectorsIndicator).toHaveTextContent( 'Failed Connectors 1' ); const failedTasksIndicator = within(metrics).getByTitle( 'Number of failed tasks' ); expect(failedTasksIndicator).toBeInTheDocument(); expect(failedTasksIndicator).toHaveTextContent('Failed Tasks 1'); }); }); }); ================================================ FILE: kafka-ui-react-app/src/components/Connect/New/New.styled.ts ================================================ import styled, { css } from 'styled-components'; export const NewConnectFormStyled = styled.form` padding: 0 16px 16px; display: flex; flex-direction: column; gap: 16px; & > button:last-child { align-self: flex-start; } `; export const Filed = styled.div<{ $hidden: boolean }>( ({ $hidden }) => $hidden && css` display: none; ` ); ================================================ FILE: kafka-ui-react-app/src/components/Connect/New/New.tsx ================================================ import React from 'react'; import { useNavigate } from 'react-router-dom'; import useAppParams from 'lib/hooks/useAppParams'; import { Controller, FormProvider, useForm } from 'react-hook-form'; import { ErrorMessage } from '@hookform/error-message'; import { yupResolver } from '@hookform/resolvers/yup'; import { clusterConnectConnectorPath, clusterConnectorsPath, ClusterNameRoute, } from 'lib/paths'; import yup from 'lib/yupExtended'; import Editor from 'components/common/Editor/Editor'; import Select from 'components/common/Select/Select'; import { FormError } from 'components/common/Input/Input.styled'; import Input from 'components/common/Input/Input'; import { Button } from 'components/common/Button/Button'; import PageHeading from 'components/common/PageHeading/PageHeading'; import Heading from 'components/common/heading/Heading.styled'; import { useConnects, useCreateConnector } from 'lib/hooks/api/kafkaConnect'; import get from 'lodash/get'; import { Connect } from 'generated-sources'; import * as S from './New.styled'; const validationSchema = yup.object().shape({ name: yup.string().required(), config: yup.string().required().isJsonObject(), }); interface FormValues { connectName: Connect['name']; name: string; config: string; } const New: React.FC = () => { const { clusterName } = useAppParams(); const navigate = useNavigate(); const { data: connects = [] } = useConnects(clusterName); const mutation = useCreateConnector(clusterName); const methods = useForm({ mode: 'all', resolver: yupResolver(validationSchema), defaultValues: { connectName: get(connects, '0.name', ''), name: '', config: '', }, }); const { handleSubmit, control, formState: { isDirty, isSubmitting, isValid, errors }, getValues, setValue, } = methods; React.useEffect(() => { if (connects && connects.length > 0 && !getValues().connectName) { setValue('connectName', connects[0].name); } }, [connects, getValues, setValue]); const onSubmit = async (values: FormValues) => { try { const connector = await mutation.createResource({ connectName: values.connectName, newConnector: { name: values.name, config: JSON.parse(values.config.trim()), }, }); if (connector) { navigate( clusterConnectConnectorPath( clusterName, connector.connect, connector.name ) ); } } catch (e) { // do nothing } }; const connectOptions = connects.map(({ name: connectName }) => ({ value: connectName, label: connectName, })); return ( Connect * (
Config ( )} />
); }; export default New; ================================================ FILE: kafka-ui-react-app/src/components/Connect/New/__tests__/New.spec.tsx ================================================ import React from 'react'; import { render, WithRoute } from 'lib/testHelpers'; import { clusterConnectConnectorPath, clusterConnectorNewPath, } from 'lib/paths'; import New from 'components/Connect/New/New'; import { connects, connector } from 'lib/fixtures/kafkaConnect'; import { fireEvent, screen, act } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { ControllerRenderProps } from 'react-hook-form'; import { useConnects, useCreateConnector } from 'lib/hooks/api/kafkaConnect'; jest.mock( 'components/common/Editor/Editor', () => (props: ControllerRenderProps) => { return