Showing preview only (7,214K chars total). Download the full file or copy to clipboard to get everything.
Repository: thatdot/quine
Branch: main
Commit: 6a744a272e04
Files: 1062
Total size: 6.6 MB
Directory structure:
gitextract_uxvox9yj/
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ └── feature_request.md
│ ├── PULL_REQUEST_TEMPLATE.md
│ ├── actions/
│ │ └── notify-slack-on-failure/
│ │ └── action.yml
│ └── workflows/
│ ├── ci.yml
│ ├── copy.bara.sky
│ └── copybara.yml
├── .gitignore
├── .scalafix.conf
├── .scalafmt.conf
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── api/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── api/
│ │ ├── codec/
│ │ │ └── SecretCodecs.scala
│ │ ├── schema/
│ │ │ └── SecretSchemas.scala
│ │ └── v2/
│ │ ├── ApiErrors.scala
│ │ ├── AwsCredentials.scala
│ │ ├── AwsRegion.scala
│ │ ├── RatesSummary.scala
│ │ ├── SaslJaasConfig.scala
│ │ ├── ShowShort.scala
│ │ ├── SuccessEnvelope.scala
│ │ ├── V2EndpointDefinitions.scala
│ │ ├── YamlCodec.scala
│ │ ├── codec/
│ │ │ ├── DisjointEither.scala
│ │ │ └── ThirdPartyCodecs.scala
│ │ ├── outputs/
│ │ │ ├── DestinationSteps.scala
│ │ │ ├── Format.scala
│ │ │ └── OutputFormat.scala
│ │ └── schema/
│ │ ├── TapirJsonConfig.scala
│ │ └── ThirdPartySchemas.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ ├── api/
│ │ ├── codec/
│ │ │ └── SecretCodecsSpec.scala
│ │ └── v2/
│ │ ├── ApiErrorsCodecSpec.scala
│ │ ├── AwsCredentialsCodecSpec.scala
│ │ ├── AwsGenerators.scala
│ │ ├── AwsRegionCodecSpec.scala
│ │ ├── ErrorResponseGenerators.scala
│ │ ├── ErrorTypeGenerators.scala
│ │ ├── SaslJaasConfigCodecSpec.scala
│ │ ├── SaslJaasConfigGenerators.scala
│ │ ├── SaslJaasConfigLoggableSpec.scala
│ │ ├── SuccessEnvelopeCodecSpec.scala
│ │ └── SuccessEnvelopeGenerators.scala
│ └── quine/
│ ├── JsonGenerators.scala
│ ├── ScalaPrimitiveGenerators.scala
│ └── TimeGenerators.scala
├── aws/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── aws/
│ │ ├── model/
│ │ │ ├── AwsCredentials.scala
│ │ │ └── AwsRegion.scala
│ │ └── util/
│ │ └── AwsOps.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── aws/
│ └── util/
│ └── AwsOpsSpec.scala
├── build.sbt
├── data/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── data/
│ │ ├── DataFoldableFrom.scala
│ │ └── DataFolderTo.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── data/
│ ├── AvroDecoderTest.scala
│ ├── DataFoldableFromSpec.scala
│ ├── DataFolderToSpec.scala
│ └── FoldableTestData.scala
├── model-converters/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── convert/
│ ├── Api2ToAws.scala
│ ├── Api2ToModel1.scala
│ ├── Api2ToOutputs2.scala
│ └── Model1ToApi2.scala
├── outputs2/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── outputs2/
│ │ ├── DestinationSteps.scala
│ │ ├── OutputEncoder.scala
│ │ ├── OutputsLoggables.scala
│ │ ├── ResultDestination.scala
│ │ ├── SaslJaasConfig.scala
│ │ ├── Sinks.scala
│ │ ├── destination/
│ │ │ ├── Drop.scala
│ │ │ ├── File.scala
│ │ │ ├── HttpEndpoint.scala
│ │ │ ├── Kafka.scala
│ │ │ ├── Kinesis.scala
│ │ │ ├── ReactiveStream.scala
│ │ │ ├── SNS.scala
│ │ │ └── StandardOut.scala
│ │ └── package.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── outputs2/
│ └── destination/
│ └── KafkaSpec.scala
├── project/
│ ├── Dependencies.scala
│ ├── Docker.scala
│ ├── Ecr.scala
│ ├── FlatcPlugin.scala
│ ├── GitVersion.scala
│ ├── Packaging.scala
│ ├── QuineSettings.scala
│ ├── ScalaFix.scala
│ ├── build.properties
│ ├── dependencySchemes.sbt
│ └── plugins.sbt
├── quine/
│ ├── recipes/
│ │ ├── apache_log.yaml
│ │ ├── apt-detection.yaml
│ │ ├── books.yaml
│ │ ├── cdn.yaml
│ │ ├── certstream-firehose.yaml
│ │ ├── conways-gol.yaml
│ │ ├── duration.yaml
│ │ ├── entity-resolution.yaml
│ │ ├── ethereum.yaml
│ │ ├── finance.yaml
│ │ ├── hpotter.yaml
│ │ ├── ingest.yaml
│ │ ├── kafka-ingest.yaml
│ │ ├── movieData.yaml
│ │ ├── pi.yaml
│ │ ├── ping.yaml
│ │ ├── pipe.yaml
│ │ ├── planetside-2.yaml
│ │ ├── quine-logs-recipe.yaml
│ │ ├── sq-test.yaml
│ │ ├── template-recipe.yaml
│ │ ├── webhook.yaml
│ │ ├── wikipedia-non-bot-revisions.yaml
│ │ └── wikipedia.yaml
│ └── src/
│ ├── main/
│ │ ├── resources/
│ │ │ ├── ionicons.tsv
│ │ │ ├── reference.conf
│ │ │ └── web/
│ │ │ ├── browserconfig.xml
│ │ │ ├── quine-ui-startup.js
│ │ │ ├── quine-ui.html
│ │ │ └── site.webmanifest
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── app/
│ │ ├── BaseApp.scala
│ │ ├── CmdArgs.scala
│ │ ├── ImproveQuine.scala
│ │ ├── Main.scala
│ │ ├── MeteredExecutors.scala
│ │ ├── Metrics.scala
│ │ ├── QuineApp.scala
│ │ ├── QuineAppIngestControl.scala
│ │ ├── QuinePreservingCodecs.scala
│ │ ├── Recipe.scala
│ │ ├── RecipeInterpreter.scala
│ │ ├── RecipeInterpreterV2.scala
│ │ ├── RecipePackage.scala
│ │ ├── RecipeV2.scala
│ │ ├── SchemaCache.scala
│ │ ├── StandingQueryResultOutput.scala
│ │ ├── StatusLines.scala
│ │ ├── config/
│ │ │ ├── Address.scala
│ │ │ ├── BaseConfig.scala
│ │ │ ├── EdgeIteration.scala
│ │ │ ├── FileAccessPolicy.scala
│ │ │ ├── FileIngestConfig.scala
│ │ │ ├── IdProviderType.scala
│ │ │ ├── MetricsConfig.scala
│ │ │ ├── MetricsReporter.scala
│ │ │ ├── PersistenceAgentType.scala
│ │ │ ├── PersistenceBuilder.scala
│ │ │ ├── PureconfigInstances.scala
│ │ │ ├── QuineConfig.scala
│ │ │ ├── QuinePersistenceBuilder.scala
│ │ │ ├── WebServerConfig.scala
│ │ │ └── errors/
│ │ │ └── ConfigErrorFormatter.scala
│ │ ├── data/
│ │ │ ├── QuineDataFoldablesFrom.scala
│ │ │ └── QuineDataFoldersTo.scala
│ │ ├── migrations/
│ │ │ ├── Migration.scala
│ │ │ ├── QuineMigrations.scala
│ │ │ └── instances/
│ │ │ ├── MultipleValuesRewrite.scala
│ │ │ └── package.scala
│ │ ├── model/
│ │ │ ├── README.md
│ │ │ ├── ingest/
│ │ │ │ ├── ContentDelimitedIngestSrcDef.scala
│ │ │ │ ├── IngestSrcDef.scala
│ │ │ │ ├── KafkaSrcDef.scala
│ │ │ │ ├── KinesisKclSrcDef.scala
│ │ │ │ ├── KinesisSrcDef.scala
│ │ │ │ ├── NamedPipeSource.scala
│ │ │ │ ├── ServerSentEventsSrcDef.scala
│ │ │ │ ├── SqsStreamSrcDef.scala
│ │ │ │ ├── WebsocketSimpleStartupSrcDef.scala
│ │ │ │ ├── serialization/
│ │ │ │ │ ├── ContentDecoder.scala
│ │ │ │ │ ├── CypherParseProtobuf.scala
│ │ │ │ │ ├── CypherToProtobuf.scala
│ │ │ │ │ ├── ImportFormat.scala
│ │ │ │ │ └── ProtobufParser.scala
│ │ │ │ └── util/
│ │ │ │ ├── AwsOps.scala
│ │ │ │ └── KafkaSettingsValidator.scala
│ │ │ ├── ingest2/
│ │ │ │ ├── V1IngestCodecs.scala
│ │ │ │ ├── V1IngestSchemas.scala
│ │ │ │ ├── V1ToV2.scala
│ │ │ │ ├── V2IngestEntities.scala
│ │ │ │ ├── V2IngestSources.scala
│ │ │ │ ├── V2ToV1.scala
│ │ │ │ ├── codec/
│ │ │ │ │ └── FrameDecoder.scala
│ │ │ │ ├── source/
│ │ │ │ │ ├── DecodedSource.scala
│ │ │ │ │ ├── FramedSource.scala
│ │ │ │ │ ├── IngestBounds.scala
│ │ │ │ │ └── QuineIngestQuery.scala
│ │ │ │ └── sources/
│ │ │ │ ├── CsvFileSource.scala
│ │ │ │ ├── FileSource.scala
│ │ │ │ ├── FramedSourceProvider.scala
│ │ │ │ ├── KafkaSource.scala
│ │ │ │ ├── KinesisKclSrc.scala
│ │ │ │ ├── KinesisSource.scala
│ │ │ │ ├── NumberIteratorSource.scala
│ │ │ │ ├── ReactiveSource.scala
│ │ │ │ ├── S3Source.scala
│ │ │ │ ├── ServerSentEventSource.scala
│ │ │ │ ├── SqsSource.scala
│ │ │ │ ├── StandardInputSource.scala
│ │ │ │ ├── WebSocketClientSource.scala
│ │ │ │ ├── WebSocketFileUploadSource.scala
│ │ │ │ └── package.scala
│ │ │ ├── outputs/
│ │ │ │ ├── ConsoleLoggingOutput.scala
│ │ │ │ ├── CypherQueryOutput.scala
│ │ │ │ ├── DropOutput.scala
│ │ │ │ ├── FileOutput.scala
│ │ │ │ ├── KafkaOutput.scala
│ │ │ │ ├── KinesisOutput.scala
│ │ │ │ ├── OutputRuntime.scala
│ │ │ │ ├── PostToEndpointOutput.scala
│ │ │ │ ├── QuinePatternOutput.scala
│ │ │ │ ├── SlackOutput.scala
│ │ │ │ └── SnsOutput.scala
│ │ │ ├── outputs2/
│ │ │ │ ├── QuineDestinationSteps.scala
│ │ │ │ ├── QuineResultDestination.scala
│ │ │ │ ├── destination/
│ │ │ │ │ ├── CypherQueryDestination.scala
│ │ │ │ │ └── Slack.scala
│ │ │ │ ├── package.scala
│ │ │ │ └── query/
│ │ │ │ ├── CypherQuery.scala
│ │ │ │ └── standing/
│ │ │ │ ├── Predicate.scala
│ │ │ │ ├── StandingQuery.scala
│ │ │ │ ├── StandingQueryPattern.scala
│ │ │ │ ├── StandingQueryResultTransformation.scala
│ │ │ │ ├── StandingQueryResultWorkflow.scala
│ │ │ │ ├── StandingQueryStats.scala
│ │ │ │ └── package.scala
│ │ │ └── transformation/
│ │ │ └── polyglot/
│ │ │ ├── Polyglot.scala
│ │ │ ├── PolyglotValueDataFoldableFrom.scala
│ │ │ ├── PolyglotValueDataFolderTo.scala
│ │ │ ├── Transformation.scala
│ │ │ └── langauges/
│ │ │ └── QuineJavaScript.scala
│ │ ├── routes/
│ │ │ ├── AdministrationRoutesImpl.scala
│ │ │ ├── AlgorithmRoutesImpl.scala
│ │ │ ├── BaseAppRoutes.scala
│ │ │ ├── DebugRoutesImpl.scala
│ │ │ ├── HealthAppRoutes.scala
│ │ │ ├── IngestApiMethods.scala
│ │ │ ├── IngestMeter.scala
│ │ │ ├── IngestRoutesImpl.scala
│ │ │ ├── IngestStreamState.scala
│ │ │ ├── IngestStreamWithControl.scala
│ │ │ ├── QueryUiConfigurationRoutesImpl.scala
│ │ │ ├── QueryUiConfigurationState.scala
│ │ │ ├── QueryUiCypherApiMethods.scala
│ │ │ ├── QueryUiRoutesImpl.scala
│ │ │ ├── QuineAppOpenApiDocs.scala
│ │ │ ├── QuineAppRoutes.scala
│ │ │ ├── StandingQueryInterfaceV2.scala
│ │ │ ├── StandingQueryRoutesV1Impl.scala
│ │ │ ├── StandingQueryStoreV1.scala
│ │ │ ├── Util.scala
│ │ │ ├── WebSocketQueryProtocolServer.scala
│ │ │ ├── exts/
│ │ │ │ ├── PekkoQuineEndpoints.scala
│ │ │ │ ├── ServerEntitiesWithExamples.scala
│ │ │ │ ├── ServerQuineEndpoints.scala
│ │ │ │ ├── ServerRequestTimeoutOps.scala
│ │ │ │ └── circe/
│ │ │ │ └── JsonEntitiesFromSchemas.scala
│ │ │ └── websocketquinepattern/
│ │ │ ├── LSPActor.scala
│ │ │ └── WebSocketQuinePatternServer.scala
│ │ ├── util/
│ │ │ ├── AtLeastOnceCypherQuery.scala
│ │ │ ├── OpenApiRenderer.scala
│ │ │ ├── QuineLoggables.scala
│ │ │ └── StringOps.scala
│ │ └── v2api/
│ │ ├── OssApiMethods.scala
│ │ ├── QuineOssV2OpenApiDocs.scala
│ │ ├── V2ApiInfo.scala
│ │ ├── V2OssRoutes.scala
│ │ ├── converters/
│ │ │ ├── Api2ToOutputs2.scala
│ │ │ ├── ApiToIngest.scala
│ │ │ ├── ApiToStanding.scala
│ │ │ ├── ApiToUiStyling.scala
│ │ │ ├── IngestToApi.scala
│ │ │ ├── UiStylingToApi.scala
│ │ │ └── package.scala
│ │ ├── definitions/
│ │ │ ├── AlgorithmApiMethods.scala
│ │ │ ├── ApiCommand.scala
│ │ │ ├── ApiUiStyling.scala
│ │ │ ├── CommonParameters.scala
│ │ │ ├── CypherApiMethods.scala
│ │ │ ├── DebugApiMethods.scala
│ │ │ ├── ParallelismParameter.scala
│ │ │ ├── QueryEffects.scala
│ │ │ ├── QuineApiMethods.scala
│ │ │ ├── QuineIdCodec.scala
│ │ │ ├── QuineIdSchemas.scala
│ │ │ ├── TapirDecodeErrorHandler.scala
│ │ │ ├── TapirRoutes.scala
│ │ │ ├── V2QueryExecutor.scala
│ │ │ ├── V2QueryWebSocketFlow.scala
│ │ │ ├── V2QuineEndpointDefinitions.scala
│ │ │ ├── ingest2/
│ │ │ │ ├── ApiIngest.scala
│ │ │ │ └── DeadLetterQueueOutput.scala
│ │ │ ├── outputs/
│ │ │ │ └── QuineDestinationSteps.scala
│ │ │ └── query/
│ │ │ └── standing/
│ │ │ ├── Predicate.scala
│ │ │ ├── StandingQuery.scala
│ │ │ ├── StandingQueryOutputStructure.scala
│ │ │ ├── StandingQueryPattern.scala
│ │ │ ├── StandingQueryResultTransformation.scala
│ │ │ ├── StandingQueryResultWorkflow.scala
│ │ │ └── StandingQueryStats.scala
│ │ └── endpoints/
│ │ ├── V2AlgorithmEndpoints.scala
│ │ ├── V2CypherEndpoints.scala
│ │ ├── V2DebugEndpoints.scala
│ │ ├── V2IngestEndpoints.scala
│ │ ├── V2QueryWebSocketEndpoints.scala
│ │ ├── V2QuineAdministrationEndpoints.scala
│ │ ├── V2StandingEndpoints.scala
│ │ ├── V2UiStylingEndpoints.scala
│ │ └── Visibility.scala
│ └── test/
│ ├── resources/
│ │ ├── addressbook.desc
│ │ ├── addressbook.proto
│ │ ├── application.conf
│ │ ├── documented_cassandra_config.conf
│ │ ├── documented_config.conf
│ │ ├── ingest_test_script/
│ │ │ ├── README.md
│ │ │ ├── ingest_test.py
│ │ │ └── requirements.txt
│ │ ├── multi_file_proto_test/
│ │ │ ├── README.md
│ │ │ ├── data/
│ │ │ │ ├── encode_examples.sh
│ │ │ │ ├── example_anyzone.binpb
│ │ │ │ ├── example_anyzone.txtpb
│ │ │ │ ├── example_zone_0.binpb
│ │ │ │ ├── example_zone_0.txtpb
│ │ │ │ ├── example_zone_1.binpb
│ │ │ │ ├── example_zone_1.txtpb
│ │ │ │ ├── example_zone_2.binpb
│ │ │ │ ├── example_zone_2.txtpb
│ │ │ │ ├── example_zone_3.binpb
│ │ │ │ └── example_zone_3.txtpb
│ │ │ └── schema/
│ │ │ ├── argus.proto
│ │ │ ├── azeroth.proto
│ │ │ ├── compile_schema.sh
│ │ │ ├── warcraft.desc
│ │ │ └── zone_rework.proto
│ │ ├── protobuf_test.binpb
│ │ ├── recipes/
│ │ │ ├── full.json
│ │ │ └── full.yaml
│ │ ├── trivial.cypher
│ │ └── yaml/
│ │ ├── invalid.yaml
│ │ ├── wikipedia-example.json
│ │ └── wikipedia-example.yaml
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── CirceCodecTestSupport.scala
│ ├── app/
│ │ ├── CmdArgsTest.scala
│ │ ├── ImproveQuineCodecSpec.scala
│ │ ├── ImproveQuineGenerators.scala
│ │ ├── QuineAppCodecSpec.scala
│ │ ├── QuineAppGenerators.scala
│ │ ├── QuineAppTelemetryTest.scala
│ │ ├── RecipeTest.scala
│ │ ├── RecipeV2Test.scala
│ │ ├── config/
│ │ │ ├── ClickHouseSecurityTest.scala
│ │ │ ├── ConfigGenerators.scala
│ │ │ ├── ConfigRoundTripSpec.scala
│ │ │ ├── QuineConfigTest.scala
│ │ │ ├── WebServerConfigTest.scala
│ │ │ └── errors/
│ │ │ └── ConfigErrorFormatterSpec.scala
│ │ ├── data/
│ │ │ ├── QuineDataFoldablesFromSpec.scala
│ │ │ └── QuineDataFoldersToSpec.scala
│ │ ├── ingest/
│ │ │ ├── DelimitedIngestSrcDefTest.scala
│ │ │ ├── KafkaSettingsValidatorTest.scala
│ │ │ ├── RawValuesIngestSrcDefTest.scala
│ │ │ ├── WritableInputStream.scala
│ │ │ └── serialization/
│ │ │ ├── ContentDecoderTest.scala
│ │ │ ├── CypherProtobufConversionsTest.scala
│ │ │ ├── ImportFormatTest.scala
│ │ │ └── ProtobufTest.scala
│ │ ├── model/
│ │ │ ├── ingest/
│ │ │ │ └── util/
│ │ │ │ └── AwsOpsSpec.scala
│ │ │ └── ingest2/
│ │ │ └── sources/
│ │ │ └── KafkaSourceSpec.scala
│ │ ├── routes/
│ │ │ ├── QueryUiCypherApiMethodsQuinePatternEnabledSpec.scala
│ │ │ ├── RouteHardeningOpsSpec.scala
│ │ │ └── websocketquinepattern/
│ │ │ ├── JsonRpcNotification.scala
│ │ │ ├── JsonRpcRequest.scala
│ │ │ ├── JsonRpcResponse.scala
│ │ │ └── WebSocketQuinePatternServerTest.scala
│ │ └── v2api/
│ │ └── definitions/
│ │ ├── ingest2/
│ │ │ ├── KafkaDlqSecretParamsSpec.scala
│ │ │ └── KafkaIngestSecretParamsSpec.scala
│ │ └── outputs/
│ │ └── KafkaDestinationSecretParamsSpec.scala
│ ├── convert/
│ │ └── Api2ToOutputs2KafkaSpec.scala
│ ├── graph/
│ │ ├── FakeQuineGraph.scala
│ │ └── StandingQueryTest.scala
│ ├── ingest2/
│ │ ├── IngestCodecSpec.scala
│ │ ├── IngestGenerators.scala
│ │ ├── IngestSourceTestSupport.scala
│ │ ├── V2IngestEntitiesCodecSpec.scala
│ │ ├── V2IngestEntitiesGenerators.scala
│ │ ├── V2IngestEntitiesPreservingCodecSpec.scala
│ │ ├── codec/
│ │ │ └── FrameDecoderSpec.scala
│ │ ├── source/
│ │ │ └── DecodedSourceSpec.scala
│ │ ├── sources/
│ │ │ ├── DelimitedSourcesSpec.scala
│ │ │ ├── FileLikeSourcesSpec.scala
│ │ │ ├── FramedSourceSpec.scala
│ │ │ └── KafkaFoldableSpec.scala
│ │ └── transformation/
│ │ ├── DataFoldableSpec.scala
│ │ ├── FoldableArbitraryHelpers.scala
│ │ └── QuineJavaScriptSpec.scala
│ ├── outputs/
│ │ ├── StandingQueryOutputCodecSpec.scala
│ │ └── StandingQueryOutputGenerators.scala
│ ├── routes/
│ │ ├── PostToEndpointSecretParamsSpec.scala
│ │ └── WriteToKafkaSecretParamsSpec.scala
│ └── v2api/
│ ├── ApiUiStylingCodecSpec.scala
│ ├── ApiUiStylingGenerators.scala
│ ├── EndpointValidationSpec.scala
│ ├── V2AlgorithmEndpointCodecSpec.scala
│ ├── V2AlgorithmEndpointGenerators.scala
│ ├── V2ApiCommonGenerators.scala
│ ├── V2CypherCodecSpec.scala
│ ├── V2CypherEndpointCodecSpec.scala
│ ├── V2CypherEndpointGenerators.scala
│ ├── V2DebugEndpointCodecSpec.scala
│ ├── V2DebugEndpointGenerators.scala
│ ├── V2IngestEndpointCodecSpec.scala
│ ├── V2IngestEndpointGenerators.scala
│ ├── V2QueryWebSocketFlowSpec.scala
│ ├── V2QuineAdministrationEndpointCodecSpec.scala
│ ├── V2QuineAdministrationEndpointGenerators.scala
│ ├── V2StandingEndpointCodecSpec.scala
│ └── V2StandingEndpointGenerators.scala
├── quine-browser/
│ ├── common.webpack.config.js
│ ├── dev/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── index.html
│ │ ├── package.json
│ │ ├── startup.js
│ │ ├── tsconfig.json
│ │ └── vite.config.ts
│ ├── dev.webpack.config.js
│ ├── prod.webpack.config.js
│ ├── src/
│ │ └── main/
│ │ ├── resources/
│ │ │ └── index.css
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── Util.scala
│ │ ├── routes/
│ │ │ ├── ClientRoutes.scala
│ │ │ ├── V2WebSocketQueryClient.scala
│ │ │ ├── WebSocketQueryClient.scala
│ │ │ └── exts/
│ │ │ └── ClientQuineEndpoints.scala
│ │ └── webapp/
│ │ ├── History.scala
│ │ ├── LaminarRoot.scala
│ │ ├── QuineInteractiveTS/
│ │ │ ├── _components/
│ │ │ │ ├── ConfigurationPortal.tsx
│ │ │ │ ├── IngestPortal.tsx
│ │ │ │ ├── QueryOutputPortal.tsx
│ │ │ │ ├── StandingQueryPortal.tsx
│ │ │ │ ├── _componentStyles.ts
│ │ │ │ └── index.ts
│ │ │ ├── _hooks/
│ │ │ │ └── useInterval.ts
│ │ │ ├── _services/
│ │ │ │ ├── adminService.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── ingestStreamService.ts
│ │ │ │ └── standingQueryService.ts
│ │ │ ├── _utils/
│ │ │ │ └── api.ts
│ │ │ ├── index.tsx
│ │ │ └── react.d.tsx
│ │ ├── QuineOssNavItems.scala
│ │ ├── Styles.scala
│ │ ├── Sugar.scala
│ │ ├── components/
│ │ │ ├── BoxPlot.scala
│ │ │ ├── ContextMenu.scala
│ │ │ ├── CypherResultsTable.scala
│ │ │ ├── HybridViewsRenderer.scala
│ │ │ ├── Loader.scala
│ │ │ ├── ManualHistogramPlot.scala
│ │ │ ├── PlotOrientation.scala
│ │ │ ├── Plotly.scala
│ │ │ ├── PlotlyFacade.scala
│ │ │ ├── RenderStrategy.scala
│ │ │ ├── StoplightElements.scala
│ │ │ ├── SunburstPlot.scala
│ │ │ ├── ToolbarButton.scala
│ │ │ ├── VisNetwork.scala
│ │ │ ├── dashboard/
│ │ │ │ ├── Card.scala
│ │ │ │ ├── CounterSummaryCard.scala
│ │ │ │ ├── MetricsDashboard.scala
│ │ │ │ ├── MetricsDashboardRenderer.scala
│ │ │ │ ├── ProgressBarMeter.scala
│ │ │ │ ├── ShardInfoCard.scala
│ │ │ │ └── TimerSummaryCard.scala
│ │ │ └── sidebar/
│ │ │ ├── CoreUISidebar.scala
│ │ │ ├── NavItem.scala
│ │ │ └── NavTitle.scala
│ │ ├── package.scala
│ │ ├── queryui/
│ │ │ ├── Counters.scala
│ │ │ ├── Event.scala
│ │ │ ├── GraphVisualization.scala
│ │ │ ├── HistoryNavigationButtons.scala
│ │ │ ├── MessageBar.scala
│ │ │ ├── PinTracker.scala
│ │ │ ├── QueryTypes.scala
│ │ │ ├── QueryUi.scala
│ │ │ ├── SvgSnapshot.scala
│ │ │ ├── TopBar.scala
│ │ │ └── VisNetworkVisualization.scala
│ │ ├── router/
│ │ │ ├── AppRouter.scala
│ │ │ ├── QuineOssPage.scala
│ │ │ ├── QuineOssRouter.scala
│ │ │ └── QuineOssRoutes.scala
│ │ ├── util/
│ │ │ └── LaminarUtils.scala
│ │ └── views/
│ │ ├── DocsV1View.scala
│ │ ├── DocsV2View.scala
│ │ ├── ExplorationUiView.scala
│ │ ├── MetricsView.scala
│ │ └── QuineOssViews.scala
│ └── tsconfig.json
├── quine-cassandra-persistor/
│ └── src/
│ ├── main/
│ │ ├── boilerplate/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── util/
│ │ │ └── TN.scala.template
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ └── cassandra/
│ │ ├── CassandraPersistor.scala
│ │ ├── DomainGraphNodes.scala
│ │ ├── DomainIndexEvents.scala
│ │ ├── Journals.scala
│ │ ├── MetaData.scala
│ │ ├── PrimeCassandraPersistor.scala
│ │ ├── QuinePatterns.scala
│ │ ├── Snapshots.scala
│ │ ├── StandingQueries.scala
│ │ ├── StandingQueryStates.scala
│ │ ├── aws/
│ │ │ ├── Journals.scala
│ │ │ ├── KeyspacesPersistor.scala
│ │ │ └── Snapshots.scala
│ │ ├── support/
│ │ │ ├── CassandraCodecs.scala
│ │ │ ├── CassandraColumn.scala
│ │ │ ├── CassandraStatementSettings.scala
│ │ │ ├── CassandraTable.scala
│ │ │ ├── TableDefinition.scala
│ │ │ └── syntax.scala
│ │ └── vanilla/
│ │ ├── CassandraPersistor.scala
│ │ ├── Journals.scala
│ │ └── Snapshots.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── CassandraPersistorSpec.scala
│ └── KeyspacesPersistorSpec.scala
├── quine-core/
│ └── src/
│ ├── main/
│ │ ├── fbs/
│ │ │ ├── base.fbs
│ │ │ ├── cypher.fbs
│ │ │ ├── domainindexevent.fbs
│ │ │ ├── journal.fbs
│ │ │ ├── snapshot.fbs
│ │ │ ├── standingquery.fbs
│ │ │ └── standingquerystates.fbs
│ │ ├── resources/
│ │ │ ├── logback.xml
│ │ │ ├── quine-pekko-overrides.conf
│ │ │ └── reference.conf
│ │ ├── scala/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ ├── exceptions/
│ │ │ │ ├── DuplicateIngestException.scala
│ │ │ │ ├── FileIngestSecurityException.scala
│ │ │ │ ├── JavaScriptException.scala
│ │ │ │ ├── KafkaValidationException.scala
│ │ │ │ ├── KinesisConfigurationError.scala
│ │ │ │ ├── NamespaceNotFoundException.scala
│ │ │ │ └── ShardIterationException.scala
│ │ │ ├── graph/
│ │ │ │ ├── AbstractNodeActor.scala
│ │ │ │ ├── AlgorithmGraph.scala
│ │ │ │ ├── ApiShutdownReason.scala
│ │ │ │ ├── BaseGraph.scala
│ │ │ │ ├── BaseNodeActor.scala
│ │ │ │ ├── BaseNodeActorView.scala
│ │ │ │ ├── CypherOpsGraph.scala
│ │ │ │ ├── DomainGraphNodeRegistry.scala
│ │ │ │ ├── EventTime.scala
│ │ │ │ ├── Expiration.scala
│ │ │ │ ├── GraphNodeHashCode.scala
│ │ │ │ ├── GraphNotReadyException.scala
│ │ │ │ ├── GraphQueryPattern.scala
│ │ │ │ ├── GraphService.scala
│ │ │ │ ├── GraphShardActor.scala
│ │ │ │ ├── IllegalHistoricalUpdate.scala
│ │ │ │ ├── LiteralOpsGraph.scala
│ │ │ │ ├── MasterStream.scala
│ │ │ │ ├── NamespaceSqIndex.scala
│ │ │ │ ├── NodeActor.scala
│ │ │ │ ├── NodeAndShardSupervisorStrategy.scala
│ │ │ │ ├── NodeEvent.scala
│ │ │ │ ├── NodeSnapshot.scala
│ │ │ │ ├── NodeWakeupFailedException.scala
│ │ │ │ ├── QuineIdProviders.scala
│ │ │ │ ├── QuineRuntimeFutureException.scala
│ │ │ │ ├── StandingQueryId.scala
│ │ │ │ ├── StandingQueryInfo.scala
│ │ │ │ ├── StandingQueryOpsGraph.scala
│ │ │ │ ├── StandingQueryResult.scala
│ │ │ │ ├── StaticNodeActorSupport.scala
│ │ │ │ ├── StaticNodeSupport.scala
│ │ │ │ ├── StaticShardGraph.scala
│ │ │ │ ├── WatchableEventType.scala
│ │ │ │ ├── behavior/
│ │ │ │ │ ├── ActorClock.scala
│ │ │ │ │ ├── AlgorithmBehavior.scala
│ │ │ │ │ ├── CypherBehavior.scala
│ │ │ │ │ ├── DomainNodeIndexBehavior.scala
│ │ │ │ │ ├── DomainNodeTests.scala
│ │ │ │ │ ├── GoToSleepBehavior.scala
│ │ │ │ │ ├── LiteralCommandBehavior.scala
│ │ │ │ │ ├── MultipleValuesStandingQueryBehavior.scala
│ │ │ │ │ ├── PriorityStashingBehavior.scala
│ │ │ │ │ ├── QuinePatternQueryBehavior.scala
│ │ │ │ │ └── StandingQueryBehavior.scala
│ │ │ │ ├── cypher/
│ │ │ │ │ ├── AggregationFunc.scala
│ │ │ │ │ ├── CompiledExpr.scala
│ │ │ │ │ ├── CompiledQuery.scala
│ │ │ │ │ ├── Exception.scala
│ │ │ │ │ ├── Expr.scala
│ │ │ │ │ ├── Func.scala
│ │ │ │ │ ├── Interpreter.scala
│ │ │ │ │ ├── MultipleValuesResultsReporter.scala
│ │ │ │ │ ├── MultipleValuesStandingQuery.scala
│ │ │ │ │ ├── MultipleValuesStandingQueryState.scala
│ │ │ │ │ ├── Proc.scala
│ │ │ │ │ ├── ProcedureExecutionLocation.scala
│ │ │ │ │ ├── Query.scala
│ │ │ │ │ ├── QueryContext.scala
│ │ │ │ │ ├── RunningCypherQuery.scala
│ │ │ │ │ ├── SkipOptimizingActor.scala
│ │ │ │ │ ├── Type.scala
│ │ │ │ │ ├── UserDefinedFunction.scala
│ │ │ │ │ ├── UserDefinedProcedure.scala
│ │ │ │ │ ├── VisitedVariableEdgeMatches.scala
│ │ │ │ │ └── quinepattern/
│ │ │ │ │ ├── QueryPlan.scala
│ │ │ │ │ ├── QueryPlanner.scala
│ │ │ │ │ ├── QueryStateBuilder.scala
│ │ │ │ │ ├── QueryStateHost.scala
│ │ │ │ │ ├── QuinePattern.scala
│ │ │ │ │ ├── QuinePatternExpressionInterpreter.scala
│ │ │ │ │ ├── QuinePatternFunction.scala
│ │ │ │ │ ├── QuinePatternHelpers.scala
│ │ │ │ │ └── procedures/
│ │ │ │ │ ├── GetFilteredEdgesProcedure.scala
│ │ │ │ │ ├── HelpBuiltinsProcedure.scala
│ │ │ │ │ ├── QuinePatternProcedure.scala
│ │ │ │ │ └── RecentNodesProcedure.scala
│ │ │ │ ├── edges/
│ │ │ │ │ ├── AbstractEdgeCollectionView.scala
│ │ │ │ │ ├── EdgeCollection.scala
│ │ │ │ │ ├── EdgeCollectionView.scala
│ │ │ │ │ ├── EdgeIndex.scala
│ │ │ │ │ ├── EdgeProcessor.scala
│ │ │ │ │ ├── MemoryFirstEdgeProcessor.scala
│ │ │ │ │ ├── PersistorFirstEdgeProcessor.scala
│ │ │ │ │ ├── ReverseOrderedEdgeCollection.scala
│ │ │ │ │ └── UnorderedEdgeCollection.scala
│ │ │ │ ├── messaging/
│ │ │ │ │ ├── AlgorithmMessage.scala
│ │ │ │ │ ├── BaseMessage.scala
│ │ │ │ │ ├── CypherMessage.scala
│ │ │ │ │ ├── ExactlyOnceAskActor.scala
│ │ │ │ │ ├── ExactlyOnceAskNodeActor.scala
│ │ │ │ │ ├── ExactlyOnceTimeoutException.scala
│ │ │ │ │ ├── GiveUpWaiting.scala
│ │ │ │ │ ├── LiteralMessage.scala
│ │ │ │ │ ├── LocalShardRef.scala
│ │ │ │ │ ├── NodeActorMailbox.scala
│ │ │ │ │ ├── QuineIdOps.scala
│ │ │ │ │ ├── QuineMessage.scala
│ │ │ │ │ ├── QuineRef.scala
│ │ │ │ │ ├── QuineRefOps.scala
│ │ │ │ │ ├── QuineResponse.scala
│ │ │ │ │ ├── ResultHandler.scala
│ │ │ │ │ ├── ShardActorMailbox.scala
│ │ │ │ │ ├── ShardMessage.scala
│ │ │ │ │ ├── ShardRef.scala
│ │ │ │ │ └── StandingQueryMessage.scala
│ │ │ │ ├── metrics/
│ │ │ │ │ ├── BinaryHistogramCounter.scala
│ │ │ │ │ ├── HostQuineMetrics.scala
│ │ │ │ │ └── implicits.scala
│ │ │ │ ├── package.scala
│ │ │ │ └── quinepattern/
│ │ │ │ ├── NonNodeActor.scala
│ │ │ │ ├── QuinePatternLoader.scala
│ │ │ │ ├── QuinePatternOpsGraph.scala
│ │ │ │ └── QuinePatternRegistry.scala
│ │ │ ├── migrations/
│ │ │ │ ├── MigrationError.scala
│ │ │ │ └── MigrationVersion.scala
│ │ │ ├── model/
│ │ │ │ ├── DGBOps.scala
│ │ │ │ ├── DomainGraphBranch.scala
│ │ │ │ ├── DomainGraphNode.scala
│ │ │ │ ├── DomainNodeEquiv.scala
│ │ │ │ ├── EdgeDirection.scala
│ │ │ │ ├── HalfEdge.scala
│ │ │ │ ├── Milliseconds.scala
│ │ │ │ ├── NodeComponents.scala
│ │ │ │ ├── PropertyValue.scala
│ │ │ │ ├── QuineIdProvider.scala
│ │ │ │ ├── QuineValue.scala
│ │ │ │ └── package.scala
│ │ │ ├── persistor/
│ │ │ │ ├── BinaryFormat.scala
│ │ │ │ ├── BloomFilteredPersistor.scala
│ │ │ │ ├── EmptyPersistor.scala
│ │ │ │ ├── ExceptionWrappingPersistenceAgent.scala
│ │ │ │ ├── InMemoryPersistor.scala
│ │ │ │ ├── IncompatibleVersion.scala
│ │ │ │ ├── PackedFlatBufferBinaryFormat.scala
│ │ │ │ ├── PartitionedPersistenceAgent.scala
│ │ │ │ ├── PersistenceAgent.scala
│ │ │ │ ├── PersistenceConfig.scala
│ │ │ │ ├── PrimePersistor.scala
│ │ │ │ ├── ShardedPersistor.scala
│ │ │ │ ├── StatelessPrimePersistor.scala
│ │ │ │ ├── UnifiedPrimePersistor.scala
│ │ │ │ ├── Version.scala
│ │ │ │ ├── WrappedPersistenceAgent.scala
│ │ │ │ └── codecs/
│ │ │ │ ├── DomainGraphNodeCodec.scala
│ │ │ │ ├── DomainIndexEventCodec.scala
│ │ │ │ ├── MultipleValuesStandingQueryStateCodec.scala
│ │ │ │ ├── NodeChangeEventCodec.scala
│ │ │ │ ├── PersistenceCodec.scala
│ │ │ │ ├── QueryPlanCodec.scala
│ │ │ │ ├── QuineValueCodec.scala
│ │ │ │ ├── SnapshotCodec.scala
│ │ │ │ └── StandingQueryCodec.scala
│ │ │ └── util/
│ │ │ ├── BaseError.scala
│ │ │ ├── Config.scala
│ │ │ ├── DeduplicationCache.scala
│ │ │ ├── ExpiringLruSet.scala
│ │ │ ├── Extractors.scala
│ │ │ ├── FromSingleExecutionContext.scala
│ │ │ ├── Funnels.scala
│ │ │ ├── FutureHelpers.scala
│ │ │ ├── FutureResult.scala
│ │ │ ├── GraphWithContextExt.scala
│ │ │ ├── Hashing.scala
│ │ │ ├── InterpM.scala
│ │ │ ├── Loggable.scala
│ │ │ ├── LoopbackPort.scala
│ │ │ ├── MonadHelpers.scala
│ │ │ ├── Packing.scala
│ │ │ ├── PekkoStreams.scala
│ │ │ ├── Pretty.scala
│ │ │ ├── ProgressCounter.scala
│ │ │ ├── QuineDispatchers.scala
│ │ │ ├── Retry.scala
│ │ │ ├── ReverseIterator.scala
│ │ │ ├── ReversibleLinkedHashSet.scala
│ │ │ ├── StringInput.scala
│ │ │ ├── StrongUUID.scala
│ │ │ ├── Tls.scala
│ │ │ ├── Valve.scala
│ │ │ └── ValveFlow.scala
│ │ └── scala-2.13/
│ │ └── scala/
│ │ └── compat/
│ │ └── CompatBuildFrom.scala
│ └── test/
│ ├── resources/
│ │ ├── application.conf
│ │ └── logback-test.xml
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── graph/
│ │ ├── ArbitraryInstances.scala
│ │ ├── DomainGraphNodeRegistryTest.scala
│ │ ├── EventTimeTest.scala
│ │ ├── GraphNodeHashCodeTest.scala
│ │ ├── GraphQueryPatternTest.scala
│ │ ├── HalfEdgeGen.scala
│ │ ├── HistoricalQueryTests.scala
│ │ ├── ScalaTestInstances.scala
│ │ ├── SerializationTests.scala
│ │ ├── StandingQueryResultTest.scala
│ │ ├── TestDataFactory.scala
│ │ ├── cypher/
│ │ │ ├── MultipleValuesResultsReporterTest.scala
│ │ │ └── quinepattern/
│ │ │ ├── OptionalStateCorrectnessTest.scala
│ │ │ ├── PR3981BugRegressionTest.scala
│ │ │ ├── PropertyAccessTest.scala
│ │ │ ├── QueryPlanRuntimeTest.scala
│ │ │ ├── QueryPlannerTest.scala
│ │ │ ├── StateInstallationTest.scala
│ │ │ └── TestPropertyAccess.scala
│ │ ├── edges/
│ │ │ ├── EdgeCollectionTests.scala
│ │ │ ├── ReverseOrderedEdgeCollectionTests.scala
│ │ │ ├── SyncEdgeCollectionTests.scala
│ │ │ └── UnorderedEdgeCollectionTests.scala
│ │ └── standing/
│ │ ├── AllPropertiesState.scala
│ │ ├── CrossStateTests.scala
│ │ ├── EdgeSubscriptionReciprocalStateTests.scala
│ │ ├── FilterMapStateTests.scala
│ │ ├── LabelsStateTests.scala
│ │ ├── LocalIdStateTests.scala
│ │ ├── LocalPropertyStateTests.scala
│ │ ├── StandingQueryStateHarness.scala
│ │ ├── SubscribeAcrossEdgeStateTests.scala
│ │ └── UnitSqStateTests.scala
│ ├── model/
│ │ ├── DomainGraphBranchTest.scala
│ │ └── DomainGraphNodeTest.scala
│ ├── persistor/
│ │ ├── InMemoryPersistorSpec.scala
│ │ ├── InvariantWrapper.scala
│ │ └── PersistenceAgentSpec.scala
│ ├── test/
│ │ ├── tagobjects/
│ │ │ └── IntegrationTest.scala
│ │ └── tags/
│ │ └── IntegrationTest.java
│ └── util/
│ ├── HexConversionsTest.scala
│ ├── LoggableTest.scala
│ ├── PackingTests.scala
│ ├── PrettyTests.scala
│ ├── SizeAndTimeBoundedCacheTest.scala
│ ├── StrongUUIDTest.scala
│ └── TestLogging.scala
├── quine-cypher/
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── graph/
│ │ │ └── cypher/
│ │ │ ├── CypherUDF.java
│ │ │ └── CypherUDP.java
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── bolt/
│ │ │ ├── Protocol.scala
│ │ │ ├── Serialization.scala
│ │ │ └── Structure.scala
│ │ ├── compiler/
│ │ │ └── cypher/
│ │ │ ├── CompM.scala
│ │ │ ├── Expression.scala
│ │ │ ├── Functions.scala
│ │ │ ├── GetFilteredEdges.scala
│ │ │ ├── Graph.scala
│ │ │ ├── ParametersIndex.scala
│ │ │ ├── Plan.scala
│ │ │ ├── Procedures.scala
│ │ │ ├── QueryPart.scala
│ │ │ ├── QueryScopeInfo.scala
│ │ │ ├── ReifyTime.scala
│ │ │ ├── StandingQueryPatterns.scala
│ │ │ ├── UncompiledQueryIdentity.scala
│ │ │ ├── Variables.scala
│ │ │ ├── WithFreeVariables.scala
│ │ │ ├── WithQuery.scala
│ │ │ └── package.scala
│ │ └── utils/
│ │ ├── CypherLoggables.scala
│ │ └── MonadVia.scala
│ └── test/
│ ├── resources/
│ │ └── application.conf
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── Bolt.scala
│ ├── BoltSerializations.scala
│ ├── CypherAggregations.scala
│ ├── CypherEquality.scala
│ ├── CypherErrors.scala
│ ├── CypherExpressions.scala
│ ├── CypherFunctions.scala
│ ├── CypherLists.scala
│ ├── CypherMatch.scala
│ ├── CypherMatchPerformance.scala
│ ├── CypherMatrix.scala
│ ├── CypherMerge.scala
│ ├── CypherRecursiveSubQuery.scala
│ ├── CypherReturn.scala
│ ├── CypherShortestPath.scala
│ ├── CypherStrings.scala
│ ├── CypherSubQueries.scala
│ ├── CypherTemporal.scala
│ ├── QueryStaticTest.scala
│ ├── SkipOptimizationsTest.scala
│ ├── VariableLengthRelationshipPattern.scala
│ └── compiler/
│ └── cypher/
│ ├── CypherComplete.scala
│ ├── CypherHarness.scala
│ ├── CypherMutate.scala
│ ├── HistoricalQueryTests.scala
│ ├── OrderedEdgesTest.scala
│ ├── SkipUninterestingNodesTest.scala
│ └── StandingQueryPatternsTest.scala
├── quine-docs/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── docs/
│ │ ├── GenerateCypherTables.scala
│ │ ├── GenerateOpenApi.scala
│ │ ├── GenerateOpenApiV2.scala
│ │ └── GenerateRecipeDirectory.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── docs/
│ ├── GenerateOpenApiTest.scala
│ └── GenerateOpenApiTestV2.scala
├── quine-endpoints/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── routes/
│ │ │ ├── AdministrationRoutes.scala
│ │ │ ├── AlgorithmRoutes.scala
│ │ │ ├── DebugOpsRoutes.scala
│ │ │ ├── IngestRoutes.scala
│ │ │ ├── QueryProtocol.scala
│ │ │ ├── QueryUiConfigurationRoutes.scala
│ │ │ ├── QueryUiRoutes.scala
│ │ │ ├── QuickQuery.scala
│ │ │ ├── StandingQueryRoutes.scala
│ │ │ └── exts/
│ │ │ ├── AnySchema.scala
│ │ │ ├── EndpointsWithCustomErrorText.scala
│ │ │ ├── EntitiesWithExamples.scala
│ │ │ └── QuineEndpoints.scala
│ │ └── v2api/
│ │ └── routes/
│ │ ├── V2MetricsRoutes.scala
│ │ ├── V2QueryUiConfigurationRoutes.scala
│ │ └── V2QueryUiRoutes.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── routes/
│ └── AwsSchemaSpec.scala
├── quine-endpoints2/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── api/
│ └── v2/
│ ├── QueryWebSocketProtocol.scala
│ ├── TapirCirceUnifiedConfig.scala
│ └── TypeDiscriminatorConfig.scala
├── quine-gremlin/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── gremlin/
│ │ ├── Exceptions.scala
│ │ ├── GremlinLexer.scala
│ │ ├── GremlinParser.scala
│ │ ├── GremlinQueryRunner.scala
│ │ ├── GremlinTypes.scala
│ │ ├── GremlinValue.scala
│ │ └── package.scala
│ └── test/
│ ├── application.conf
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── gremlin/
│ ├── ErrorMessages.scala
│ ├── GremlinHarness.scala
│ └── SimpleQueries.scala
├── quine-language/
│ └── src/
│ ├── main/
│ │ ├── antlr4/
│ │ │ └── Cypher.g4
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── language/
│ │ │ ├── server/
│ │ │ │ ├── QuineLanguageServer.java
│ │ │ │ └── QuineTextDocumentService.java
│ │ │ └── testclient/
│ │ │ └── QuineLanguageClient.java
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── cypher/
│ │ │ ├── CollectingErrorListener.scala
│ │ │ ├── ast/
│ │ │ │ └── AST.scala
│ │ │ ├── phases/
│ │ │ │ ├── LexerPhase.scala
│ │ │ │ ├── MaterializationPhase.scala
│ │ │ │ ├── ParserPhase.scala
│ │ │ │ └── SymbolAnalysis.scala
│ │ │ ├── utils/
│ │ │ │ └── Helpers.scala
│ │ │ └── visitors/
│ │ │ ├── ast/
│ │ │ │ ├── CreateVisitor.scala
│ │ │ │ ├── EffectVisitor.scala
│ │ │ │ ├── ForeachVisitor.scala
│ │ │ │ ├── InQueryCallVisitor.scala
│ │ │ │ ├── MatchClauseVisitor.scala
│ │ │ │ ├── MultiPartQueryVisitor.scala
│ │ │ │ ├── NodeLabelVisitor.scala
│ │ │ │ ├── PatternVisitor.scala
│ │ │ │ ├── ProjectionBodyVisitor.scala
│ │ │ │ ├── ProjectionItemVisitor.scala
│ │ │ │ ├── QueryVisitor.scala
│ │ │ │ ├── ReadingClauseVisitor.scala
│ │ │ │ ├── RegularQueryVisitor.scala
│ │ │ │ ├── ReturnVisitor.scala
│ │ │ │ ├── SetItemVisitor.scala
│ │ │ │ ├── SinglePartQueryVisitor.scala
│ │ │ │ ├── SingleQueryVisitor.scala
│ │ │ │ ├── UnwindClauseVisitor.scala
│ │ │ │ ├── UpdatingClauseVisitor.scala
│ │ │ │ ├── WhereClauseVisitor.scala
│ │ │ │ ├── WithVisitor.scala
│ │ │ │ ├── expressions/
│ │ │ │ │ ├── AddSubtractVisitor.scala
│ │ │ │ │ ├── AndVisitor.scala
│ │ │ │ │ ├── AtomVisitor.scala
│ │ │ │ │ ├── ComparisonVisitor.scala
│ │ │ │ │ ├── DoubleVisitor.scala
│ │ │ │ │ ├── ExpressionVisitor.scala
│ │ │ │ │ ├── FunctionInvocationVisitor.scala
│ │ │ │ │ ├── IntegerVisitor.scala
│ │ │ │ │ ├── LiteralVisitor.scala
│ │ │ │ │ ├── MapLiteralVisitor.scala
│ │ │ │ │ ├── MultiplyDivideModuloVisitor.scala
│ │ │ │ │ ├── NonArithmeticOperatorVisitor.scala
│ │ │ │ │ ├── NumberVisitor.scala
│ │ │ │ │ ├── OrExpressionVisitor.scala
│ │ │ │ │ ├── ParameterVisitor.scala
│ │ │ │ │ ├── PartialComparisonVisitor.scala
│ │ │ │ │ ├── PowerOfVisitor.scala
│ │ │ │ │ ├── PropertyVisitor.scala
│ │ │ │ │ ├── StringListNullVisitor.scala
│ │ │ │ │ ├── UnaryAddSubtractVisitor.scala
│ │ │ │ │ ├── VariableVisitor.scala
│ │ │ │ │ └── XorVisitor.scala
│ │ │ │ └── patterns/
│ │ │ │ ├── AnonymousPatternPartVisitor.scala
│ │ │ │ ├── MatchPatternVisitor.scala
│ │ │ │ ├── NodePatternVisitor.scala
│ │ │ │ ├── PatternElementChainVisitor.scala
│ │ │ │ ├── PatternElementVisitor.scala
│ │ │ │ ├── PatternExpVisitor.scala
│ │ │ │ ├── PatternPartVisitor.scala
│ │ │ │ └── RelationshipPatternVisitor.scala
│ │ │ └── semantic/
│ │ │ ├── AddSubtractVisitor.scala
│ │ │ ├── AndVisitor.scala
│ │ │ ├── AnonymousPatternPartVisitor.scala
│ │ │ ├── AtomVisitor.scala
│ │ │ ├── ComparisonVisitor.scala
│ │ │ ├── CreateVisitor.scala
│ │ │ ├── DoubleVisitor.scala
│ │ │ ├── ExpressionVisitor.scala
│ │ │ ├── FunctionInvocationVisitor.scala
│ │ │ ├── InQueryCallVisitor.scala
│ │ │ ├── IntegerVisitor.scala
│ │ │ ├── LiteralVisitor.scala
│ │ │ ├── MapLiteralVisitor.scala
│ │ │ ├── MatchClauseVisitor.scala
│ │ │ ├── MultiPartQueryVisitor.scala
│ │ │ ├── MultiplyDivideModuloVisitor.scala
│ │ │ ├── NodeLabelVisitor.scala
│ │ │ ├── NodePatternVisitor.scala
│ │ │ ├── NonArithmeticOperatorVisitor.scala
│ │ │ ├── NotVisitor.scala
│ │ │ ├── NumberVisitor.scala
│ │ │ ├── OrExpressionVisitor.scala
│ │ │ ├── ParameterVisitor.scala
│ │ │ ├── PartialComparisonVisitor.scala
│ │ │ ├── PatternElementChainVisitor.scala
│ │ │ ├── PatternElementVisitor.scala
│ │ │ ├── PatternPartVisitor.scala
│ │ │ ├── PatternVisitor.scala
│ │ │ ├── PowerOfVisitor.scala
│ │ │ ├── ProjectionBodyVisitor.scala
│ │ │ ├── ProjectionItemVisitor.scala
│ │ │ ├── PropertyExpressionVisitor.scala
│ │ │ ├── QueryVisitor.scala
│ │ │ ├── ReadingClauseVisitor.scala
│ │ │ ├── RegularQueryVisitor.scala
│ │ │ ├── RelationshipPatternVisitor.scala
│ │ │ ├── ReturnVisitor.scala
│ │ │ ├── SetItemVisitor.scala
│ │ │ ├── SinglePartQueryVisitor.scala
│ │ │ ├── SingleQueryVisitor.scala
│ │ │ ├── StringListNullVisitor.scala
│ │ │ ├── UnaryAddOrSubtractVisitor.scala
│ │ │ ├── UnwindClauseVisitor.scala
│ │ │ ├── UpdatingClauseVisitor.scala
│ │ │ ├── VariableVisitor.scala
│ │ │ ├── WhereVisitor.scala
│ │ │ └── XorVisitor.scala
│ │ └── language/
│ │ ├── Cypher.scala
│ │ ├── ast/
│ │ │ └── AST.scala
│ │ ├── diagnostic/
│ │ │ └── Diagnostic.scala
│ │ ├── domain/
│ │ │ └── Graph.scala
│ │ ├── phases/
│ │ │ ├── CompilerPhase.scala
│ │ │ ├── CompilerState.scala
│ │ │ ├── Phase.scala
│ │ │ ├── TypeCheckingPhase.scala
│ │ │ └── Upgrade.scala
│ │ ├── prettyprint/
│ │ │ ├── ASTInstances.scala
│ │ │ ├── BaseInstances.scala
│ │ │ ├── CypherASTInstances.scala
│ │ │ ├── PrettyPrint.scala
│ │ │ ├── ResultInstances.scala
│ │ │ ├── SymbolAnalysisInstances.scala
│ │ │ ├── TypeInstances.scala
│ │ │ └── package.scala
│ │ ├── semantic/
│ │ │ └── Semantics.scala
│ │ ├── server/
│ │ │ ├── ContextAwareLanguageService.scala
│ │ │ ├── Helpers.scala
│ │ │ └── SimpleTrie.scala
│ │ ├── testclient/
│ │ │ ├── QuineLanguageClient.scala
│ │ │ └── TestProgram.scala
│ │ └── types/
│ │ └── Type.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── cypher/
│ │ ├── phases/
│ │ │ ├── LexerPhaseTest.scala
│ │ │ ├── ParserPhaseTest.scala
│ │ │ └── PhaseCompositionTest.scala
│ │ └── visitors/
│ │ └── ast/
│ │ ├── AddSubtractVisitorTests.scala
│ │ ├── AndVisitorTests.scala
│ │ ├── AtomVisitorTests.scala
│ │ ├── ComparisonVisitorTests.scala
│ │ ├── DoubleVisitorTests.scala
│ │ ├── ExpressionVisitorTests.scala
│ │ ├── IntegerVisitorTests.scala
│ │ ├── LiteralVisitorTests.scala
│ │ ├── MapLiteralVisitorTests.scala
│ │ ├── MultiplyDivideModuloVisitorTests.scala
│ │ ├── NonArithmeticOperatorVisitorTests.scala
│ │ ├── NumberVisitorTests.scala
│ │ ├── OrExpressionVisitorTests.scala
│ │ ├── ParameterVisitorTests.scala
│ │ ├── PartialComparisonVisitorTests.scala
│ │ ├── PowerOfVisitorTests.scala
│ │ ├── PropertyVisitorTests.scala
│ │ ├── UnaryAddSubtractVisitorTests.scala
│ │ └── XorVisitorTests.scala
│ └── language/
│ ├── diagnostic/
│ │ └── DiagnosticTest.scala
│ ├── parser/
│ │ └── ParserTests.scala
│ ├── phases/
│ │ ├── AlphaRenamingTests.scala
│ │ ├── MaterializationTests.scala
│ │ ├── PipelineExplorer.scala
│ │ └── SymbolAnalysisTests.scala
│ ├── prettyprint/
│ │ └── PrettyPrintTest.scala
│ ├── semantics/
│ │ └── SemanticAnalysisTests.scala
│ ├── server/
│ │ ├── ContextAwareLanguageServiceTest.scala
│ │ ├── LanguageServerHelper.scala
│ │ ├── QuineLanguageServerTest.scala
│ │ ├── QuineTextDocumentServiceTest.scala
│ │ └── TextDocumentServiceHelper.scala
│ └── types/
│ ├── GraphElementTypeTests.scala
│ ├── TypeCheckerTests.scala
│ ├── TypeEntryDuplicateTest.scala
│ └── TypeSystemTest.scala
├── quine-mapdb-persistor/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ ├── MapDbGlobalPersistor.scala
│ │ └── MapDbPersistor.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── MapDbPersistorSpec.scala
│ └── MapDbPersistorTests.scala
├── quine-rocksdb-persistor/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ ├── RocksDbPersistor.scala
│ │ └── RocksDbPrimePersistor.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── RocksDbKeyEncodingTest.scala
│ ├── RocksDbPersistorSpec.scala
│ └── RocksDbPersistorTests.scala
├── quine-serialization/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── serialization/
│ ├── AvroSchemaCache.scala
│ ├── EncoderDecoder.scala
│ ├── ProtobufSchemaCache.scala
│ ├── QuineValueToProtobuf.scala
│ ├── SchemaError.scala
│ └── data/
│ ├── QuineSerializationFoldablesFrom.scala
│ └── QuineSerializationFoldersTo.scala
├── visnetwork-facade/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── visnetwork/
│ ├── DataSet.scala
│ ├── Events.scala
│ ├── Network.scala
│ └── package.scala
└── vite-shared/
├── base.config.ts
├── fixtures/
│ ├── metrics.ts
│ ├── query-results.ts
│ └── ui-config.ts
├── index.ts
├── package.json
├── plugins/
│ ├── mock-api-factory.ts
│ └── serve-scalajs-bundle.ts
├── tsconfig.json
└── utils/
└── mime-types.ts
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
================================================
FILE: .github/PULL_REQUEST_TEMPLATE.md
================================================
# Description
Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
Fixes # (issue)
## Type of change
Please delete options that are not relevant.
- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] This change requires a documentation update
# How Has This Been Tested?
Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
- [ ] Test A
- [ ] Test B
**Test Configuration**:
* Firmware version:
* Hardware:
* Toolchain:
* SDK:
# Checklist:
- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream modules
================================================
FILE: .github/actions/notify-slack-on-failure/action.yml
================================================
name: Notify Slack on CI Failure
description: Posts a failure notification to Slack via incoming webhook
inputs:
job-name:
description: Human-readable name of the failed job
required: true
webhook-url:
description: Slack incoming webhook URL
required: true
runs:
using: composite
steps:
- name: Notify Slack
uses: slackapi/slack-github-action@v2.1.0
with:
webhook: ${{ inputs.webhook-url }}
webhook-type: incoming-webhook
payload: |
{
"text": "CI ${{ inputs.job-name }} failed on ${{ github.repository }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":red_circle: *CI ${{ inputs.job-name }} failed*\n*Repo:* ${{ github.repository }}\n*Branch:* main\n*Commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run>"
}
}
]
}
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
concurrency:
group: ci-${{ github.head_ref }}
cancel-in-progress: true
on:
push:
branches:
- main
pull_request:
workflow_dispatch:
env:
JAVA_OPTS: >-
-Xms4096M -Xmx4096M -Xss6M
-Dfile.encoding=UTF-8
--add-opens java.base/java.lang=ALL-UNNAMED
jobs:
test_scala:
name: Test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v3
name: Cache Embedded Cassandra
with:
path: |
~/.embedded-cassandra
key: cassandra-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/**') }}
- uses: coursier/cache-action@v6
with:
extraKey: '2.13'
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: Set up scala
uses: sbt/setup-sbt@159bc2bcdce6cc8f23f9faa80a0efc07632b17b9
- run: sbt -v test quine/assembly quine-docs/generateDocs 'scalafixAll --check'
- name: Notify Slack on failure
if: failure() && github.ref == 'refs/heads/main'
uses: ./.github/actions/notify-slack-on-failure
with:
job-name: Test
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
scalafmt:
name: Scalafmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: coursier/cache-action@v6
with:
extraKey: 'fmt'
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: Set up scala
uses: sbt/setup-sbt@159bc2bcdce6cc8f23f9faa80a0efc07632b17b9
- run: sbt -v scalafmtCheckAll scalafmtSbtCheck
- name: Notify Slack on failure
if: failure() && github.ref == 'refs/heads/main'
uses: ./.github/actions/notify-slack-on-failure
with:
job-name: Scalafmt
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
================================================
FILE: .github/workflows/copy.bara.sky
================================================
SOT_REPO = "git@github.com:thatdot/quine-plus.git"
SOT_BRANCH = "main"
DESTINATION_REPO = "git@github.com:thatdot/quine.git"
DESTINATION_BRANCH = "main"
COMMITTER = "thatbot-copy[bot] <98922356+thatbot-copy[bot]@users.noreply.github.com>"
LOCAL_SOT = "file:///usr/src/app"
PROJECT_LEVEL_INCLUDE = [
"build.properties",
"Dependencies.scala",
"dependencySchemes.sbt",
"FlatcPlugin.scala",
"Packaging.scala",
"GitVersion.scala",
"Docker.scala",
"Ecr.scala",
"ParadoxThatdot.scala",
"plugins.sbt",
"QuineSettings.scala",
"ScalaFix.scala"
]
PUSH_INCLUDE = [
"public/**",
".github/workflows/copybara.yml",
".github/workflows/copy.bara.sky",
".scalafix.conf",
".scalafmt.conf",
".gitignore",
] + ["project/" + f for f in PROJECT_LEVEL_INCLUDE]
PUSH_EXCLUDE = []
PUSH_TRANSFORMATIONS = [
]
PR_INCLUDE = ["**"]
PR_EXCLUDE = []
PR_TRANSFORMATIONS = [
core.move("", "public", paths = glob(["**"])),
core.move("public/.github/workflows/", ".github/workflows/", paths = glob(["copybara.yml", "copy.bara.sky"])),
core.move("public/.scalafix.conf", ".scalafix.conf"),
core.move("public/.scalafmt.conf", ".scalafmt.conf"),
core.move("public/.gitignore", ".gitignore"),
core.move("public/project/", "project/", paths = glob(PROJECT_LEVEL_INCLUDE)),
]
SCRUB_MESSAGE = [
# Replace anything beginning "ENTERPRISE:" (until "PUBLIC:" if present, or else to the end of the message with \z)
metadata.scrubber("ENTERPRISE:\\s(?:.|\n)*?(?:PUBLIC:\\s|\\z)"),
# Best effort to remove references to internal PRs that will be dead links publicly
metadata.scrubber(" \\(#\\d+\\)$"),
# remove any QU-XXXX numbers on their own lines (case insensitive)
metadata.scrubber("^[\\r\\f ]*[qQ][uU]-\\d+[\\r\\f ]*\\n"),
]
def cancel_after_frozen(ctx):
ctx.console.verbose("TODO add a way to freeze private copies of PRs")
if False:
return ctx.console.error("Internal copy of PR is write-protected")
else:
return ctx.success()
# Push workflow
core.workflow(
name = "push",
origin = git.origin(
url = LOCAL_SOT if LOCAL_SOT else SOT_REPO,
ref = SOT_BRANCH,
),
destination = git.github_destination(
url = DESTINATION_REPO,
push = DESTINATION_BRANCH,
),
origin_files = glob(PUSH_INCLUDE, exclude = PUSH_EXCLUDE),
authoring = authoring.pass_thru(default = COMMITTER),
mode = "ITERATIVE",
transformations = SCRUB_MESSAGE + [
metadata.restore_author("ORIGINAL_AUTHOR", search_all_changes = True),
metadata.expose_label("COPYBARA_INTEGRATE_REVIEW"),
] + (PUSH_TRANSFORMATIONS if PUSH_TRANSFORMATIONS else core.reverse(PR_TRANSFORMATIONS)),
)
# Init workflow
core.workflow(
name = "initialize",
origin = git.origin(
url = LOCAL_SOT if LOCAL_SOT else SOT_REPO,
ref = SOT_BRANCH,
),
destination = git.github_destination(
url = DESTINATION_REPO,
push = DESTINATION_BRANCH,
),
origin_files = glob(PUSH_INCLUDE, exclude = PUSH_EXCLUDE),
authoring = authoring.pass_thru(default = COMMITTER),
mode = "SQUASH",
transformations = [metadata.use_last_change()] + core.reverse(PR_TRANSFORMATIONS),
)
# Pull Request workflow
core.workflow(
name = "pr",
origin = git.github_pr_origin( # NB will not accept PRs with submodules
url = DESTINATION_REPO,
branch = DESTINATION_BRANCH,
),
destination = git.github_pr_destination(
url = SOT_REPO,
destination_ref = SOT_BRANCH,
integrates = [],
),
destination_files = glob(PUSH_INCLUDE, exclude = PUSH_EXCLUDE),
origin_files = glob(PR_INCLUDE if PR_INCLUDE else ["**"], exclude = PR_EXCLUDE),
authoring = authoring.pass_thru(default = COMMITTER),
mode = "CHANGE_REQUEST",
set_rev_id = False,
transformations = [
cancel_after_frozen,
metadata.save_author("ORIGINAL_AUTHOR"),
metadata.expose_label("GITHUB_PR_NUMBER", new_name = "Closes", separator = DESTINATION_REPO.replace("git@github.com:", " ").replace(".git", "#")),
] + PR_TRANSFORMATIONS,
)
================================================
FILE: .github/workflows/copybara.yml
================================================
name: Copy Commits to thatdot/quine Repo
on:
push:
branches:
- main
pull_request_target:
workflow_dispatch:
inputs:
copybaraArgs:
description: "Arguments to be passed to the copybara agent"
required: false
default: ""
type: string
copybaraWorkflow:
description: "Which copybara action to run"
required: false
type: choice
options:
- ""
- initialize
- push
- pr
jobs:
clone-code:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Cache Docker registry
uses: actions/cache@v5
with:
path: /tmp/docker-registry
key: universal
- name: Generate token
id: generate-token
uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 (archived)
with:
app_id: 169359 # thatBot Copy Application on thatBot account
private_key: ${{ secrets.THATBOT_COPY_KEY }}
- name: Update cached Copybara docker image
run: |
docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2 && npx wait-on tcp:5000
docker pull localhost:5000/copybara:latest || docker pull public.ecr.aws/p0a2o6c9/copybara:latest
docker tag public.ecr.aws/p0a2o6c9/copybara:latest localhost:5000/copybara:latest && docker push localhost:5000/copybara:latest || true
- name: Run Copybara
uses: thatdot/copybara-action@73152945ea4bb6c57b3d68e74787bdc1f80392ab # main@2024-05-13
with:
copybara_image: localhost:5000/copybara # thatdot-managed copy of olivr/copybara (itself a build of https://github.com/google/copybara/blob/master/Dockerfile)
copybara_image_tag: latest
custom_config: ".github/workflows/copy.bara.sky"
copybara_options: ${{ github.event.inputs.copybaraArgs }}
workflow: ${{ github.event.inputs.copybaraWorkflow }}
ssh_key: ${{ secrets.COPYBARA_SSH_KEY }}
access_token: ${{ steps.generate-token.outputs.token }}
committer: "thatbot-copy[bot] <98922356+thatbot-copy[bot]@users.noreply.github.com>" # INV: should match COMMITTER in copy.bara.sky -- this one is used for the commits, the one in there is used as a default author
sot_branch: main
sot_repo: thatdot/quine-plus
destination_branch: main
destination_repo: thatdot/quine
================================================
FILE: .gitignore
================================================
*.db
*.data
*.class
# likely binaries
quine-*.jar
novelty-*.jar
# local settings
.java-version
.jvmopts
local.sbt
local.conf
**/secret.conf
# bloop and metals
.bloop
.bsp
# metals
project/metals.sbt
.metals
# vs code
.vscode
# sbt
project/project/
project/target/
target/
.sbtopts
# virtual machine crash logs (http://www.java.com/en/download/help/error_hotspot.xml)
hs_err_pid*
replay_pid*
# eclipse
build/
.classpath
.project
.settings
.worksheet
bin/
.cache
# intellij idea
*.iml
*.ipr
*.iws
.idea
# mac
.DS_Store
# python test
public/quine/src/test/resources/ingest_test_script/venv/
quine/src/test/resources/ingest_test_script/venv/
# file mirroring management
public/project/
# vim
*.swp
*.swo
# Snyk
.dccache
# run/test outputs
metrics-logs
# exclusions
!/scripts/build/
!/lib/dasho-annotations.jar
# SBOM files
*.bom.json
# Playwright MCP Artifacts
.playwright-mcp
# npm/node (Vite dev workspaces - development only)
node_modules/
/package-lock.json
.vite
================================================
FILE: .scalafix.conf
================================================
// .scalafix.conf
rules = [
OrganizeImports
ExplicitResultTypes
LeakingImplicitClassVal
DisableSyntax
// "github:ohze/scalafix-rules/FinalObject"
]
OrganizeImports {
groupedImports = AggressiveMerge
groups = [
"re:javax?\\.", // a re: prefix denotes a regex, this will group java. and javax. packages together
"scala.",
"org.apache.pekko.",
"*",
"com.thatdot."
]
}
// Prohibit auto-derivation imports that bypass explicit codec configuration.
// https://docs.google.com/document/d/1E5MaCuRZ4F1wCx3lI9FmZFIYC8Ov5TFU8DBMj-THAsk/
// explains why we insist on explicit codec configuration.
DisableSyntax.regex = [
{
id = "noCirceAuto"
pattern = "import io\\.circe\\.generic.*\\.auto\\..*"
message = "Prohibited: `io.circe.generic.[extras.]auto` is too slow and problematic. Use `io.circe.generic.[extras.]semiauto.derive[Configured](De|En)coder` instead (probably with `V2ApiConfiguration.typeDiscriminatorConfig.asCirce` in scope)."
},
{
id = "noTapirAuto"
pattern = "import sttp\\.tapir\\.generic\\.auto\\..*"
message = "Prohibited: `sttp.tapir.generic.auto` is too slow and problematic. Use explicit `Schema.derived` or `deriveSchema` with proper configuration."
},
{
id = "noPureconfigAuto"
pattern = "import pureconfig\\.generic\\.auto\\._"
message = "Prohibited: `pureconfig.generic.auto._` causes shapeless macro explosion. Use `pureconfig.generic.semiauto.deriveConvert` with explicit ConfigConvert instances for nested types."
}
]
================================================
FILE: .scalafmt.conf
================================================
version = 2.7.5 // scala-steward:off
// Additional style conventions not enforced by scalafmt:
// - mark `case class`es and `case object`s `final` wherever possible
// - prefer `sealed abstract class` over `sealed trait` wherever possible
// - when in doubt, https://nrinaudo.github.io/scala-best-practices/ has sensible recommendations
maxColumn = 120
align.preset = none
continuationIndent {
callSite = 2
defnSite = 2
ctorSite = 2
}
newlines.afterCurlyLambda = preserve
literals.float = Upper
literals.hexDigits = Upper
trailingCommas = always
rewrite.rules = [
RedundantBraces,
RedundantParens,
SortModifiers,
PreferCurlyFors,
]
unindentTopLevelOperators = true
indentOperator.preset = akka
project.excludeFilters=[".*/com/thatdot/quine/app/util/OpenApiRenderer\\.scala"]
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Code of Conduct
## Code of Conduct for the Quine Community
thatDot is dedicated to providing the best community possible for the Quine community. Our goal is to provide the opportunity for community participants to learn, communicate, contribute and collaborate. The Community Code of Conduct governs how we all participate and behave. As such we are committed to creating a diverse, harassment-free experience for everyone, regardless of gender, sexual orientation, disability, physical appearance, body size, race, or religion. We do not tolerate harassment of community participants in any form. Any form of written, social or verbal communication that can be offensive or harassing to any community member, participant or staff is not allowed. Community participants violating these rules may be sanctioned or expelled from the community.
## Expectations for All Community Members
### Be kind.
All community participants should feel welcome, regardless of their personal background. Please be polite, courteous, and considerate to fellow participants. No offensive comments regarding to gender, sexual orientation, disability, physical appearance, body size, race, or religion will be tolerated.
### Be respectful.
We expect all participants to be respectful when communicating with other participants, even when differences of opinion arise. Participants are expected to work together to resolve disagreements constructively and respectfully. Disagreement is no excuse for poor manners. Please be patient.
### Reach out and ask for help.
Please inform our community operator or forum moderator if you feel a violation has taken place and our staff will address the situation. Ask questions if you are unsure and be helpful to those who ask. You can also contact [community@quine.io](mailto:community@quine.io)
### Communicate and collaborate.
The concept of the community is based on working together and participants will gain the most from the community by actively participating and communicating effectively. As such, we encourage collaboration and communication as long as they are conducted in a positive and constructive way.
### Continue.
This list is not exhaustive or complete. Please use your own good judgement on proper behavior and contribute to a productive and pleasant community experience.
## How To Report Inappropriate Behavior
If a community participant engages in harassing behavior, community staff may take any action they consider appropriate, including expulsion from the community. If you are being harassed or know of someone else is being harassed, please inform our community staff immediately by contacting [community@quine.io](mailto:community@quine.io)
We expect participants to abide by these rules at all community-related activities. Thank you for your cooperation.
## Privacy Policy
We understand that privacy is important to our community participants and users of these products and services. Our [privacy policy](https://www.thatdot.com/privacy/) explains how we collect, use, share, and protect personal information.
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing
The community is the heart of all open-source projects. We welcome contributions from all people and strive to build a welcoming and open community of contributors, users, participants, speakers, lurkers, and anyone else who comes by.
## Code of Conduct
All community members must be good citizens; be sure to read the [Code of Conduct](https://github.com/streaminggraph/recipes/blob/main/code-of-conduct.md) page to understand what this means.
## Contributing Code
Code contributions can be made through Github. We welcome all contributors and any improvements to Quine, the website, recipes, etc.
## Contribution License
All contributions to the Quine repository are released under the same license as the Quine project overall. For details, see the license in the Github repository.
================================================
FILE: LICENSE
================================================
MIT License with Commons Clause
Copyright © 2014 Ryan Wright; © 2019 thatDot, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"Commons Clause" License Condition v1.0
The Software is provided to you by the Licensor under the License, as defined
below, subject to the following condition.
Without limiting other conditions in the License, the grant of rights under
the License will not include, and the License does not grant to you, right to
Sell the Software.
For purposes of the foregoing, "Sell" means practicing any or all of the
rights granted to you under the License to provide to third parties, for a fee
or other consideration (including without limitation fees for hosting or
consulting/support services related to the Software), a product or service
whose value derives, entirely or substantially, from the functionality of the
Software. Any license notice or attribution required by the License must also
include this Commons Cause License Condition notice.
Software: Quine
License: MIT
Licensor: thatDot, Inc.
================================================
FILE: README.md
================================================
[](https://github.com/thatdot/quine/releases)
[](https://hub.docker.com/r/thatdot/quine)
[](https://that.re/quine-slack)
<div style="padding-top: 10px;">
<div style="vertical-align:middle;">
<img width="400" height="100%" src="https://quine.io/assets/images/quine_logo.svg">
</div>
<div style="vertical-align:middle;">
<p>Quine is a streaming graph interpreter; a server-side program that consumes data, builds it into a stateful graph structure, and runs live computation on that graph to answer questions or compute results. Those results stream out in real-time.</p>
</div>
</div>
You interact with Quine by connecting it to a data source (Kafka, Kinesis Data Stream, SQS, files, stdin, etc.) and using regular database queries to stream that data in, build the graph structure, and find important patterns.
Three design choices define Quine, setting it apart from all event stream processing systems:
1. A graph-structured data model
2. An asynchronous actor-based graph computational model
3. Standing queries
Standing queries live inside the graph and automatically propagate the incremental results computed from both historical data and new streaming data. Once matches are found, standing queries trigger actions using those results (e.g., execute code, transform other data in the graph, publish data to another system like Apache Kafka or Kinesis).
All together, Quine can:
* Consume high-volume streaming event data
* Convert it into durable, versioned, connected data
* Monitor that connected data for complex structures or values
* Trigger arbitrary computation on the event of each match
This collection of capabilities is profoundly powerful! It represents a complete system for stateful event-driven arbitrary computation in a platform scalable to any size of data or desired throughput.
Read the docs at [quine.io](https://quine.io) to learn more.
## Building from source
In order to build Quine locally, you'll need to have the following installed:
* A recent version of the Java Development Kit (17 or newer)
* The [`sbt` build tool](https://www.scala-sbt.org/download.html)
* Yarn 0.22.0+ (for frontend components of `quine-browser` subproject)
Then:
```
sbt compile # compile all projects
sbt test # compile and run all projects' tests
sbt fixall # reformat and lint all source files
sbt quine/run # to build and run Quine
sbt quine/assembly # assemble Quine into a jar
```
## Launch Quine:
Run Quine from an executable `.jar` file built from this repo or downloaded from the repo [releases](https://github.com/thatdot/quine/releases) page.
```shell
❯ java -jar quine-x.x.x.jar -h
Quine universal program
Usage: quine [options]
-W, --disable-web-service
disable Quine web service
-p, --port <value> web service port (default is 8080)
-r, --recipe name, file, or URL
follow the specified recipe
-x, --recipe-value key=value
recipe parameter substitution
--force-config disable recipe configuration defaults
--no-delete disable deleting data file when process exits
-h, --help
-v, --version print Quine program version
```
For example, to run the [Wikipedia page ingest](https://quine.io/recipes/wikipedia/) getting started recipe:
``` shell
❯ java -jar quine-x.x.x.jar -r wikipedia
```
With Docker installed, run Quine from Docker Hub.
``` shell
❯ docker run -p 8080:8080 thatdot/quine
```
The [quick start](https://quine.io/getting-started/quick-start/) guide will get you up and running the first time, ingesting data, and submitting your first query.
## Quine Recipes
Quine recipes are a great way to get started developing with Quine. A recipe is a document that contains all the information necessary for Quine to execute any data processing task. Ingest data from batch sources like `.json` or `.csv` files hosted locally, or connect to streaming data sources from Kafka or Kinesis.
[Recipes](https://quine.io/components/recipe-ref-manual/) are `yaml` documents containing the configuration for components including:
* [Ingest Streams](https://quine.io/components/ingest-sources/) to read streaming data from sources and update graph data
* [Standing Queries](https://quine.io/components/standing-queries/) to transform graph data, and to produce aggregates and other outputs
* UI configuration to specialize the web user interface for the use-case that is the subject of the Recipe
Please see [Quine's Recipe repository](https://quine.io/recipes/) for a list of available Recipes. Or create your own and contribute it back to the community for others to use.
## Contributing to Quine
The community is the heart of all open-source projects. We welcome contributions from all people and strive to build a welcoming and open community of contributors, users, participants, speakers, and lurkers. Everyone is welcome.
More information is included in our [contribution](https://github.com/thatdot/quine/blob/main/CONTRIBUTING.md) guidelines.
================================================
FILE: api/src/main/scala/com/thatdot/api/codec/SecretCodecs.scala
================================================
package com.thatdot.api.codec
import io.circe.{Decoder, Encoder}
import com.thatdot.common.security.Secret
/** Circe codecs for [[Secret]] values. */
object SecretCodecs {
/** Encoder that uses `Secret.toString` for redaction.
* This is the default encoder and should be used for HTTP API responses.
*/
implicit val secretEncoder: Encoder[Secret] = Encoder.encodeString.contramap(_.toString)
/** Creates an encoder that preserves the actual value for persistence and cluster communication.
* Requires a witness (`import Secret.Unsafe._`) to call, making the intent explicit.
* WARNING: Only use this encoder for internal storage paths, never for external HTTP responses.
* This method is intentionally NOT implicit to prevent accidental use in API contexts.
*/
def preservingEncoder(implicit ev: Secret.UnsafeAccess): Encoder[Secret] =
Encoder.encodeString.contramap(_.unsafeValue)
/** Decoder that wraps incoming strings in a Secret. */
implicit val secretDecoder: Decoder[Secret] = Decoder.decodeString.map(Secret(_))
}
================================================
FILE: api/src/main/scala/com/thatdot/api/schema/SecretSchemas.scala
================================================
package com.thatdot.api.schema
import sttp.tapir.Schema
import com.thatdot.common.security.Secret
/** Tapir schemas for [[Secret]] values. */
object SecretSchemas {
/** Schema that represents Secret as a string in OpenAPI.
*
* The schema maps Secret to/from String using Secret.apply for creation
* and Secret.toString for serialization (which redacts the value).
*/
implicit val secretSchema: Schema[Secret] =
Schema.string.map((s: String) => Some(Secret(s)))(_.toString)
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/ApiErrors.scala
================================================
package com.thatdot.api.v2
import java.util.UUID
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.model.StatusCode
import sttp.tapir.{EndpointOutput, Schema, statusCode}
import com.thatdot.api.v2.TypeDiscriminatorConfig.instances._
import com.thatdot.api.v2.schema.TapirJsonConfig.jsonBody
import com.thatdot.common.logging.Log._
import com.thatdot.quine.util.BaseError
/** Errors that api v2 cares to distinguish for reporting */
sealed trait ErrorType {
val message: String
}
/** The types of errors that the api knows how to distinguish and report
*
* Should be extended for all errors we want to be distinguished in an api response.
* See: [[BaseError]] for future extension.
*/
object ErrorType {
/** General Api error that we don't have any extra information about */
case class ApiError(message: String) extends ErrorType
object ApiError {
implicit lazy val schema: Schema[ApiError] = Schema.derived
implicit val encoder: Encoder[ApiError] = deriveConfiguredEncoder
implicit val decoder: Decoder[ApiError] = deriveConfiguredDecoder
}
/** Api error type for any sort of Decode Failure
*
* Used currently for a custom decode failure handler passed to Pekko Server Options.
*/
case class DecodeError(message: String, help: Option[String] = None) extends ErrorType
object DecodeError {
implicit lazy val schema: Schema[DecodeError] = Schema.derived
implicit val encoder: Encoder[DecodeError] = deriveConfiguredEncoder
implicit val decoder: Decoder[DecodeError] = deriveConfiguredDecoder
}
/** Api error type for any Cypher Error
*
* This could be further broken down based upon CypherException later.
*/
case class CypherError(message: String) extends ErrorType
object CypherError {
implicit lazy val schema: Schema[CypherError] = Schema.derived
implicit val encoder: Encoder[CypherError] = deriveConfiguredEncoder
implicit val decoder: Decoder[CypherError] = deriveConfiguredDecoder
}
implicit lazy val schema: Schema[ErrorType] = Schema.derived
implicit val encoder: Encoder[ErrorType] = deriveConfiguredEncoder
implicit val decoder: Decoder[ErrorType] = deriveConfiguredDecoder
}
trait HasErrors extends Product with Serializable {
def errors: List[ErrorType]
}
/** Provides the types of error codes that the api can give back to a user.
*
* Maps directly to http error codes (400s to 500s)
* They are combined with Coproduct from shapeless where used. This should be updated to Union in scala 3.
*/
object ErrorResponse {
case class ServerError(errors: List[ErrorType]) extends HasErrors
case class BadRequest(errors: List[ErrorType]) extends HasErrors
case class NotFound(errors: List[ErrorType]) extends HasErrors
case class Unauthorized(errors: List[ErrorType]) extends HasErrors
case class ServiceUnavailable(errors: List[ErrorType]) extends HasErrors
implicit private val errorListSchema: Schema[List[ErrorType]] = ErrorType.schema.asIterable[List]
object ServerError {
def apply(error: String): ServerError = ServerError(List(ErrorType.ApiError(error)))
def apply(error: ErrorType): ServerError = ServerError(List(error))
def apply(error: BaseError): ServerError = ServerError(
List(ErrorType.ApiError(error.getMessage)),
)
def ofErrors(errors: List[BaseError]): ServerError = ServerError(
errors.map(err => ErrorType.ApiError(err.getMessage)),
)
implicit lazy val schema: Schema[ServerError] = Schema.derived
implicit val encoder: Encoder[ServerError] = deriveConfiguredEncoder
implicit val decoder: Decoder[ServerError] = deriveConfiguredDecoder
}
// It would be nice to take away the below methods once we have our errors properly coded.
object BadRequest {
def apply(error: String): BadRequest = BadRequest(List(ErrorType.ApiError(error)))
def apply(error: ErrorType): BadRequest = BadRequest(List(error))
def apply(error: BaseError): BadRequest = BadRequest(List(ErrorType.ApiError(error.getMessage)))
def ofErrorStrings(errors: List[String]): BadRequest = BadRequest(errors.map(err => ErrorType.ApiError(err)))
def ofErrors(errors: List[BaseError]): BadRequest = BadRequest(
errors.map(err => ErrorType.ApiError(err.getMessage)),
)
implicit lazy val schema: Schema[BadRequest] = Schema.derived
implicit val encoder: Encoder[BadRequest] = deriveConfiguredEncoder
implicit val decoder: Decoder[BadRequest] = deriveConfiguredDecoder
}
object NotFound {
def apply(error: String): NotFound = NotFound(List(ErrorType.ApiError(error)))
def apply(error: ErrorType): NotFound = NotFound(List(error))
def apply(error: BaseError): NotFound = NotFound(List(ErrorType.ApiError(error.getMessage)))
def ofErrors(errors: List[BaseError]): NotFound = NotFound(errors.map(err => ErrorType.ApiError(err.getMessage)))
implicit lazy val schema: Schema[NotFound] = Schema.derived
implicit val encoder: Encoder[NotFound] = deriveConfiguredEncoder
implicit val decoder: Decoder[NotFound] = deriveConfiguredDecoder
}
object Unauthorized {
def apply(reason: String): Unauthorized = Unauthorized(List(ErrorType.ApiError(reason)))
def apply(reason: ErrorType) = new Unauthorized(List(reason))
implicit lazy val schema: Schema[Unauthorized] = Schema.derived
implicit val encoder: Encoder[Unauthorized] = deriveConfiguredEncoder
implicit val decoder: Decoder[Unauthorized] = deriveConfiguredDecoder
implicit val loggable: AlwaysSafeLoggable[Unauthorized] = unauthorized =>
s"Unauthorized: ${unauthorized.errors.mkString("[", ", ", "]")}"
}
object ServiceUnavailable {
def apply(error: String): ServiceUnavailable = ServiceUnavailable(List(ErrorType.ApiError(error)))
def apply(error: ErrorType): ServiceUnavailable = ServiceUnavailable(List(error))
def apply(error: BaseError): ServiceUnavailable = ServiceUnavailable(List(ErrorType.ApiError(error.getMessage)))
def ofErrors(errors: List[BaseError]): ServiceUnavailable = ServiceUnavailable(
errors.map(err => ErrorType.ApiError(err.getMessage)),
)
implicit lazy val schema: Schema[ServiceUnavailable] = Schema.derived
implicit val encoder: Encoder[ServiceUnavailable] = deriveConfiguredEncoder
implicit val decoder: Decoder[ServiceUnavailable] = deriveConfiguredDecoder
}
}
object ErrorResponseHelpers extends LazySafeLogging {
/** Default error catching for server logic. Could use a second look once more errors are codified */
def toServerError(e: Throwable)(implicit logConfig: LogConfig): ErrorResponse.ServerError = {
val correlationId = UUID.randomUUID().toString
logger.error(log"Internal server error [correlationId=${Safe(correlationId)}]" withException e)
ErrorResponse.ServerError(
s"An internal error occurred. Reference ID: $correlationId",
)
}
/** Convert IllegalArgumentException to BadRequest with the exception's message */
def toBadRequest(e: IllegalArgumentException): ErrorResponse.BadRequest =
ErrorResponse.BadRequest(e.getMessage)
def serverError(possibleReasons: String*)(implicit
enc: Encoder[ErrorResponse.ServerError],
dec: Decoder[ErrorResponse.ServerError],
sch: Schema[ErrorResponse.ServerError],
): EndpointOutput[ErrorResponse.ServerError] =
statusCode(StatusCode.InternalServerError).and {
jsonBody[ErrorResponse.ServerError]
.description(ErrorText.serverErrorDescription(possibleReasons: _*))
}
def badRequestError(possibleReasons: String*)(implicit
enc: Encoder[ErrorResponse.BadRequest],
dec: Decoder[ErrorResponse.BadRequest],
sch: Schema[ErrorResponse.BadRequest],
): EndpointOutput[ErrorResponse.BadRequest] =
statusCode(StatusCode.BadRequest).and {
jsonBody[ErrorResponse.BadRequest]
.description(ErrorText.badRequestDescription(possibleReasons: _*))
}
def notFoundError(possibleReasons: String*)(implicit
enc: Encoder[ErrorResponse.NotFound],
dec: Decoder[ErrorResponse.NotFound],
sch: Schema[ErrorResponse.NotFound],
): EndpointOutput[ErrorResponse.NotFound] =
statusCode(StatusCode.NotFound).and {
jsonBody[ErrorResponse.NotFound]
.description(ErrorText.notFoundDescription(possibleReasons: _*))
}
def unauthorizedError(possibleReasons: String*)(implicit
enc: Encoder[ErrorResponse.Unauthorized],
dec: Decoder[ErrorResponse.Unauthorized],
sch: Schema[ErrorResponse.Unauthorized],
): EndpointOutput[ErrorResponse.Unauthorized] =
statusCode(StatusCode.Unauthorized).and {
jsonBody[ErrorResponse.Unauthorized]
.description(ErrorText.unauthorizedErrorDescription(possibleReasons: _*))
}
}
object ErrorText {
private def notFoundDoc =
"""Not Found
|
|The resource referenced was not found.
|
|%s
|
|""".stripMargin
private def badRequestDoc =
s"""Bad Request
|
| Something in your request is invalid, and could not be processed.
| Review your request and attempt to submit it again.
|
| %s
|
| Contact support if you continue to have issues.
|
|""".stripMargin
private val serverErrorDoc =
s"""Internal Server Error
|
| Encountered an unexpected condition that prevented processing your request.
|
| %s
|
| Contact support if you continue to have issues.
|
|""".stripMargin
private val unauthorizedDoc =
s"""Unauthorized
|
|Permission to access a protected resource not found
|
|%s
|
|""".stripMargin
/** Manually generate a markdown bullet list from the list of message strings. */
private def buildErrorMessage(docs: String, messages: Seq[String]): String =
if (messages.isEmpty) docs.format("")
else {
val bulletSeparator = "\n - "
val msgString = f"Possible reasons:$bulletSeparator${messages.mkString(bulletSeparator)}"
docs.format(msgString)
}
def badRequestDescription(messages: String*): String =
buildErrorMessage(badRequestDoc, messages)
def notFoundDescription(messages: String*): String =
buildErrorMessage(notFoundDoc, messages)
def serverErrorDescription(messages: String*): String =
buildErrorMessage(serverErrorDoc, messages)
def unauthorizedErrorDescription(messages: String*): String =
buildErrorMessage(unauthorizedDoc, messages)
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/AwsCredentials.scala
================================================
package com.thatdot.api.v2
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.{description, encodedExample, title}
import com.thatdot.api.codec.SecretCodecs
import com.thatdot.api.v2.TypeDiscriminatorConfig.instances.circeConfig
import com.thatdot.common.security.Secret
@title("AWS Credentials")
@description(
"Explicit AWS access key and secret to use. If not provided, defaults to environmental credentials according to the default AWS credential chain. See: <https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default>.",
)
final case class AwsCredentials(
@encodedExample("ATIAXNKBTSB57V2QF11X")
accessKeyId: Secret,
@encodedExample("MDwbQe5XT4uOA3jQB/FhPaZpJdFkW13ryAL29bAk")
secretAccessKey: Secret,
)
object AwsCredentials {
import com.thatdot.api.codec.SecretCodecs.{secretEncoder, secretDecoder}
/** Encoder that redacts credential values for API responses. */
implicit val encoder: Encoder[AwsCredentials] = deriveConfiguredEncoder
implicit val decoder: Decoder[AwsCredentials] = deriveConfiguredDecoder
implicit lazy val schema: Schema[AwsCredentials] = {
import com.thatdot.api.schema.SecretSchemas.secretSchema
Schema.derived
}
/** Encoder that preserves credential values for persistence.
* Requires witness (`import Secret.Unsafe._`) to call.
*/
def preservingEncoder(implicit ev: Secret.UnsafeAccess): Encoder[AwsCredentials] =
PreservingCodecs.encoder
}
/** Separate object to avoid implicit scope pollution. */
private object PreservingCodecs {
def encoder(implicit ev: Secret.UnsafeAccess): Encoder[AwsCredentials] = {
// Shadow the redacting encoder with the preserving version
implicit val secretEncoder: Encoder[Secret] = SecretCodecs.preservingEncoder
deriveConfiguredEncoder
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/AwsRegion.scala
================================================
package com.thatdot.api.v2
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.{description, encodedExample, title}
@title("AWS Region")
@description(
"AWS region code. e.g. `us-west-2`. If not provided, defaults according to the default AWS region provider chain. See: <https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/region-selection.html#automatically-determine-the-aws-region-from-the-environment>.",
)
final case class AwsRegion(
@encodedExample("us-west-2")
region: String,
)
object AwsRegion {
implicit val encoder: Encoder[AwsRegion] = Encoder.encodeString.contramap(_.region)
implicit val decoder: Decoder[AwsRegion] = Decoder.decodeString.map(AwsRegion(_))
implicit lazy val schema: Schema[AwsRegion] = Schema.string[AwsRegion]
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/RatesSummary.scala
================================================
package com.thatdot.api.v2
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.{description, title}
import com.thatdot.api.v2.TypeDiscriminatorConfig.instances.circeConfig
@title("Rates Summary")
@description("Summary statistics about a metered rate.")
final case class RatesSummary(
@description("Number of items metered") count: Long,
@description("Approximate rate per second in the last minute") oneMinute: Double,
@description("Approximate rate per second in the last five minutes") fiveMinute: Double,
@description("Approximate rate per second in the last fifteen minutes") fifteenMinute: Double,
@description("Approximate rate per second since the meter was started") overall: Double,
)
object RatesSummary {
implicit val encoder: Encoder[RatesSummary] = deriveConfiguredEncoder
implicit val decoder: Decoder[RatesSummary] = deriveConfiguredDecoder
implicit lazy val schema: Schema[RatesSummary] = Schema.derived
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/SaslJaasConfig.scala
================================================
package com.thatdot.api.v2
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
import com.thatdot.api.codec.SecretCodecs
import com.thatdot.api.codec.SecretCodecs._
import com.thatdot.api.schema.SecretSchemas._
import com.thatdot.api.v2.TypeDiscriminatorConfig.instances.circeConfig
import com.thatdot.common.logging.Log.AlwaysSafeLoggable
import com.thatdot.common.security.Secret
/** SASL/JAAS configuration for Kafka authentication.
*
* Represents the structured form of Kafka's `sasl.jaas.config` property. Each subtype
* corresponds to a specific SASL mechanism supported by Kafka.
*
* @see [[https://kafka.apache.org/41/security/authentication-using-sasl Kafka SASL Authentication]]
*/
sealed trait SaslJaasConfig
object SaslJaasConfig {
implicit val encoder: Encoder[SaslJaasConfig] = deriveConfiguredEncoder
implicit val decoder: Decoder[SaslJaasConfig] = deriveConfiguredDecoder
implicit lazy val schema: Schema[SaslJaasConfig] = Schema.derived
/** Encoder that preserves credential values for persistence.
* Requires witness (`import Secret.Unsafe._`) to call.
*/
def preservingEncoder(implicit ev: Secret.UnsafeAccess): Encoder[SaslJaasConfig] = {
// Shadow the redacting encoder with the preserving version
implicit val secretEncoder: Encoder[Secret] = SecretCodecs.preservingEncoder
// Derive encoders for subtypes that contain secrets
implicit val plainLoginEncoder: Encoder[PlainLogin] = deriveConfiguredEncoder
implicit val scramLoginEncoder: Encoder[ScramLogin] = deriveConfiguredEncoder
implicit val oauthBearerLoginEncoder: Encoder[OAuthBearerLogin] = deriveConfiguredEncoder
deriveConfiguredEncoder
}
/** Format a SASL/JAAS configuration as a Kafka JAAS config string.
*
* @param config
* the SASL/JAAS configuration to format
* @param renderSecret
* function to render secret values (e.g., redact or expose)
* @return
* a JAAS configuration string
*/
private def formatJaasString(config: SaslJaasConfig, renderSecret: Secret => String): String = config match {
case PlainLogin(username, password) =>
s"""org.apache.kafka.common.security.plain.PlainLoginModule required username="$username" password="${renderSecret(
password,
)}";"""
case ScramLogin(username, password) =>
s"""org.apache.kafka.common.security.scram.ScramLoginModule required username="$username" password="${renderSecret(
password,
)}";"""
case OAuthBearerLogin(clientId, clientSecret, scope, tokenEndpointUrl) =>
val scopePart = scope.map(s => s""" scope="$s"""").getOrElse("")
val tokenUrlPart = tokenEndpointUrl.map(u => s""" sasl.oauthbearer.token.endpoint.url="$u"""").getOrElse("")
s"""org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required clientId="$clientId" clientSecret="${renderSecret(
clientSecret,
)}"$scopePart$tokenUrlPart;"""
}
/** Loggable instance for SaslJaasConfig that outputs JAAS format with redacted secrets.
*
* Produces output in Kafka's native JAAS config string format, making logs directly
* comparable to Kafka documentation and examples. Passwords and client secrets are
* shown as "****".
*/
implicit val logSaslJaasConfig: AlwaysSafeLoggable[SaslJaasConfig] =
formatJaasString(_, _ => "****")
/** Convert a SASL/JAAS configuration to Kafka's JAAS config string format.
*
* Requires an unsafe access witness to extract the secret values.
*
* @param config
* the SASL/JAAS configuration to convert
* @param ev
* witness that the caller has acknowledged unsafe access to secrets
* @return
* a JAAS configuration string suitable for Kafka's `sasl.jaas.config` property
*/
def toJaasConfigString(config: SaslJaasConfig)(implicit ev: Secret.UnsafeAccess): String =
formatJaasString(config, _.unsafeValue)
}
/** PLAIN authentication mechanism for Kafka SASL.
*
* Uses simple username/password authentication. The password is transmitted in cleartext
* (though typically over TLS), so this mechanism should only be used with SSL/TLS encryption.
*
* Corresponds to Kafka's `org.apache.kafka.common.security.plain.PlainLoginModule`.
*
* @param username
* SASL username for authentication
* @param password
* SASL password (redacted in API responses and logs)
* @see [[https://kafka.apache.org/41/security/authentication-using-sasl/#authentication-using-saslplain Kafka SASL/PLAIN]]
*/
final case class PlainLogin(
username: String,
password: Secret,
) extends SaslJaasConfig
object PlainLogin {
implicit val encoder: Encoder[PlainLogin] = deriveConfiguredEncoder
implicit val decoder: Decoder[PlainLogin] = deriveConfiguredDecoder
implicit lazy val schema: Schema[PlainLogin] = Schema.derived
}
/** SCRAM (Salted Challenge Response Authentication Mechanism) for Kafka SASL.
*
* A more secure alternative to PLAIN that does not transmit the password in cleartext.
* Kafka supports SCRAM-SHA-256 and SCRAM-SHA-512 variants.
*
* Corresponds to Kafka's `org.apache.kafka.common.security.scram.ScramLoginModule`.
*
* @param username
* SASL username for authentication
* @param password
* SASL password (redacted in API responses and logs)
* @see [[https://kafka.apache.org/41/security/authentication-using-sasl/#authentication-using-saslscram Kafka SASL/SCRAM]]
*/
final case class ScramLogin(
username: String,
password: Secret,
) extends SaslJaasConfig
object ScramLogin {
implicit val encoder: Encoder[ScramLogin] = deriveConfiguredEncoder
implicit val decoder: Decoder[ScramLogin] = deriveConfiguredDecoder
implicit lazy val schema: Schema[ScramLogin] = Schema.derived
}
/** OAuth Bearer authentication mechanism for Kafka SASL.
*
* Uses OAuth 2.0 client credentials flow to obtain access tokens for Kafka authentication.
* The client authenticates with the OAuth provider using client ID and secret, then uses
* the resulting token to authenticate with Kafka.
*
* Corresponds to Kafka's `org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule`.
*
* @param clientId
* OAuth 2.0 client identifier
* @param clientSecret
* OAuth 2.0 client secret (redacted in API responses and logs)
* @param scope
* Optional OAuth scope(s) to request
* @param tokenEndpointUrl
* Optional OAuth token endpoint URL (if not using OIDC discovery)
* @see [[https://kafka.apache.org/41/security/authentication-using-sasl/#authentication-using-sasloauthbearer Kafka SASL/OAUTHBEARER]]
*/
final case class OAuthBearerLogin(
clientId: String,
clientSecret: Secret,
scope: Option[String] = None,
tokenEndpointUrl: Option[String] = None,
) extends SaslJaasConfig
object OAuthBearerLogin {
implicit val encoder: Encoder[OAuthBearerLogin] = deriveConfiguredEncoder
implicit val decoder: Decoder[OAuthBearerLogin] = deriveConfiguredDecoder
implicit lazy val schema: Schema[OAuthBearerLogin] = Schema.derived
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/ShowShort.scala
================================================
package com.thatdot.api.v2
trait ShowShort[-A] {
def showShort(a: A): String
}
trait ShowShortOps {
implicit class ShortShower[A: ShowShort](a: A) {
def showShort: String = ShowShort[A].showShort(a)
}
}
object ShowShort {
def apply[A](implicit instance: ShowShort[A]): ShowShort[A] = instance
implicit def eitherShowShort[A: ShowShort, B: ShowShort]: ShowShort[Either[A, B]] =
(eitherAB: Either[A, B]) => eitherAB.fold(ShowShort[A].showShort, ShowShort[B].showShort)
implicit def hasErrorsShowShort[A <: HasErrors]: ShowShort[A] =
(hasErrors: A) => s"[${hasErrors.errors.map(_.message).mkString(", ")}]"
object syntax extends ShowShortOps
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/SuccessEnvelope.scala
================================================
package com.thatdot.api.v2
import io.circe.generic.extras.Configuration
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
sealed trait SuccessEnvelope[+Content]
sealed trait CreatedOrNoContent[+Content] extends SuccessEnvelope[Content]
sealed trait CreatedOrOk[+Content] extends SuccessEnvelope[Content]
object SuccessEnvelope {
implicit private val defaultConfig: Configuration = Configuration.default.withDefaults
case class Ok[Content](content: Content, message: Option[String] = None, warnings: List[String] = Nil)
extends SuccessEnvelope[Content]
with CreatedOrOk[Content]
object Ok {
implicit def schema[A](implicit inner: Schema[A]): Schema[Ok[A]] = Schema.derived
implicit def encoder[A: Encoder]: Encoder[Ok[A]] = deriveConfiguredEncoder
implicit def decoder[A: Decoder]: Decoder[Ok[A]] = deriveConfiguredDecoder
}
case class Created[Content](content: Content, message: Option[String] = None, warnings: List[String] = Nil)
extends SuccessEnvelope[Content]
with CreatedOrNoContent[Content]
with CreatedOrOk[Content]
object Created {
implicit def schema[A](implicit inner: Schema[A]): Schema[Created[A]] = Schema.derived
implicit def encoder[A: Encoder]: Encoder[Created[A]] = deriveConfiguredEncoder
implicit def decoder[A: Decoder]: Decoder[Created[A]] = deriveConfiguredDecoder
}
case class Accepted(
message: String = "Request accepted. Starting to process task.",
monitorUrl: Option[String] = None,
) extends SuccessEnvelope[Nothing]
object Accepted {
implicit lazy val schema: Schema[Accepted] = Schema.derived
implicit val encoder: Encoder[Accepted] = deriveConfiguredEncoder
implicit val decoder: Decoder[Accepted] = deriveConfiguredDecoder
}
case object NoContent extends SuccessEnvelope[Nothing] with CreatedOrNoContent[Nothing] {
implicit lazy val schema: Schema[NoContent.type] = Schema.derived
implicit val encoder: Encoder[NoContent.type] = Encoder.encodeUnit.contramap(_ => ())
implicit val decoder: Decoder[NoContent.type] = Decoder.decodeUnit.map(_ => NoContent)
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/V2EndpointDefinitions.scala
================================================
package com.thatdot.api.v2
import java.nio.charset.{Charset, StandardCharsets}
import java.util.concurrent.TimeUnit
import scala.concurrent.duration.FiniteDuration
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}
import io.circe.{Decoder, Encoder}
import shapeless.ops.coproduct.{Basis, CoproductToEither, Inject}
import shapeless.{:+:, CNil, Coproduct}
import sttp.tapir.CodecFormat.TextPlain
import sttp.tapir.DecodeResult.Value
import sttp.tapir._
import com.thatdot.api.v2.ErrorResponse.{BadRequest, ServerError}
import com.thatdot.api.v2.ErrorResponseHelpers.{toBadRequest, toServerError}
import com.thatdot.api.v2.schema.TapirJsonConfig
import com.thatdot.common.logging.Log._
import com.thatdot.common.quineid.QuineId
import com.thatdot.quine.model.{Milliseconds, QuineIdProvider}
trait V2EndpointDefinitions extends TapirJsonConfig with LazySafeLogging {
implicit protected def logConfig: LogConfig
type AtTime = Milliseconds
// ------- id ----------------
protected def toQuineId(s: String): DecodeResult[QuineId] =
idProvider.qidFromPrettyString(s) match {
case Success(id) => Value(id)
case Failure(_) => DecodeResult.Error(s, new IllegalArgumentException(s"'$s' is not a valid QuineId"))
}
// TODO Use Tapir Validator IdProvider.validate
val idProvider: QuineIdProvider
implicit val quineIdCodec: Codec[String, QuineId, TextPlain] =
Codec.string.mapDecode(toQuineId)(idProvider.qidToPrettyString)
/** Since timestamps get encoded as milliseconds since 1970 in the REST API,
* it is necessary to define the serialization/deserialization to/from a long.
*/
protected def toAtTime(rawTime: Long): DecodeResult[AtTime] = {
val now = System.currentTimeMillis
if (rawTime > now)
DecodeResult.Error(rawTime.toString, new IllegalArgumentException(s"Times in the future are not supported."))
else Value(Milliseconds(rawTime))
}
/** Schema for an at time */
implicit val atTimeEndpointCodec: Codec[String, AtTime, TextPlain] = Codec.long.mapDecode(toAtTime)(_.millis)
val atTimeParameter: EndpointInput.Query[Option[AtTime]] =
query[Option[AtTime]]("atTime")
.description(
"An integer timestamp in milliseconds since the Unix epoch representing the historical moment to query.",
)
// ------ timeout -------------
implicit val timeoutCodec: Codec[String, FiniteDuration, TextPlain] =
Codec.long.mapDecode(l => DecodeResult.Value(FiniteDuration(l, TimeUnit.MILLISECONDS)))(_.toMillis)
val timeoutParameter: EndpointInput.Query[FiniteDuration] =
query[FiniteDuration]("timeout")
.description("Milliseconds to wait before the HTTP request times out.")
.default(FiniteDuration.apply(20, TimeUnit.SECONDS))
type EndpointBase = Endpoint[Unit, Unit, ServerError, Unit, Any]
/** Base for api/v2 endpoints with common errors
*
* @param basePaths Provided base Paths will be appended in order, i.e. `endpoint("a","b") == /api/v2/a/b`
*/
def rawEndpoint(
basePaths: String*,
): Endpoint[Unit, Unit, Nothing, Unit, Any] =
infallibleEndpoint
.in(basePaths.foldLeft("api" / "v2")((path, segment) => path / segment))
def yamlBody[T]()(implicit
schema: Schema[T],
encoder: Encoder[T],
decoder: Decoder[T],
): EndpointIO.Body[String, T] = stringBodyAnyFormat(YamlCodec.createCodec[T](), StandardCharsets.UTF_8)
def jsonOrYamlBody[T](tOpt: Option[T] = None)(implicit
schema: Schema[T],
encoder: Encoder[T],
decoder: Decoder[T],
): EndpointIO.OneOfBody[T, T] = tOpt match {
case None => oneOfBody[T](jsonBody[T], yamlBody[T]())
case Some(t) =>
oneOfBody[T](jsonBody[T].example(t), yamlBody[T]().example(t))
}
def textBody[T](codec: Codec[String, T, TextPlain]): EndpointIO.Body[String, T] =
stringBodyAnyFormat(codec, Charset.defaultCharset())
/** Used to produce an endpoint that only has ServerErrors that are caught here.
*
* - Wraps server logic in tapir endpoints for catching any exception and lifting to ServerError(500 code).
*/
def recoverServerError[In, Out](
fa: Future[In],
)(outToResponse: In => Out): Future[Either[ServerError, Out]] = {
implicit val ec: ExecutionContext = ExecutionContext.parasitic
fa.map(out => Right(outToResponse(out))).recover(t => Left(toServerError(t)))
}
/** Recover from errors that could cause the provided future to fail. Errors are represented as any shape Coproduct
*
* - Wraps server logic in tapir endpoints for catching any exception and lifting to ServerError(500 code).
* - Used when the input error type, `Err`, is itself a Coproduct that does not contain ServerError.
* - The Left of the output Either will itself be a nested either with all coproduct elements accounted for.
* This is used for tapir endpoint definition as the errorOut type
* - When the Coproduct has size greater than 2 the tapir Either and CoproductToEither is swapped.
* to fix this map the errorOut to be swapped for the endpoint: `_.mapErrorOut(err => err.swap)(err => err.swap)`
*/
def recoverServerErrorEither[In, Out, Err <: Coproduct](
fa: Future[Either[Err, In]],
)(outToResponse: In => Out)(implicit
basis: Basis[ServerError :+: Err, Err],
c2e: CoproductToEither[ServerError :+: Err],
): Future[Either[c2e.Out, Out]] = {
implicit val ec: ExecutionContext = ExecutionContext.parasitic
fa.map {
case Left(err) => Left(c2e(err.embed[ServerError :+: Err]))
case Right(value) => Right(outToResponse(value))
}.recover(t => Left(c2e(Coproduct[ServerError :+: Err](toServerError(t)))))
}
/** Recover from errors that could cause the provided future to fail. Errors are represented as a Coproduct
* with ServerError explicitly the head of the Coproduct `Err` in the provided Future.
*
* - Wraps server logic in tapir endpoints for catching any exception and lifting to ServerError(500 code).
* - Used when the input error type, `Err`, is itself a Coproduct that does contain ServerError
* - The Left of the output Either will itself be a nested either with all coproduct elements accounted for.
* This is used for tapir endpoint definition as the errorOut type
* - When the Coproduct has size greater than 2 the tapir Either and CoproductToEither is swapped.
* to fix this map the errorOut to be swapped for the endpoint: `_.mapErrorOut(err => err.swap)(err => err.swap)`
*/
def recoverServerErrorEitherWithServerError[In, Out, Err <: Coproduct](
fa: Future[Either[ServerError :+: Err, In]],
)(outToResponse: In => Out)(implicit
basis: Basis[ServerError :+: Err, ServerError :+: Err],
c2e: CoproductToEither[ServerError :+: Err],
): Future[Either[c2e.Out, Out]] = {
implicit val ec: ExecutionContext = ExecutionContext.parasitic
fa.map {
case Left(err) => Left(c2e(err.embed[ServerError :+: Err]))
case Right(value) => Right(outToResponse(value))
}.recover(t => Left(c2e(Coproduct[ServerError :+: Err](toServerError(t)))))
}
/** Recover from errors that could cause the provided future to fail. Errors should likely not be represented
* as a Coproduct in the input provided Future
*
* - Wraps server logic in tapir endpoints for catching any exception and lifting to ServerError(500 code).
* - Used when the input error type, `Err`, is not a Coproduct itself.
* - The Left of the output Either will itself be an Either[ServerError, Err] with all coproduct elements accounted for.
* This is used for tapir endpoint definition as the errorOut type
*/
def recoverServerErrorEitherFlat[In, Out, Err](
fa: Future[Either[Err, In]],
)(outToResponse: In => Out)(implicit
c2e: CoproductToEither[ServerError :+: Err :+: CNil],
): Future[Either[c2e.Out, Out]] = {
implicit val ec: ExecutionContext = ExecutionContext.parasitic
fa.map {
case Left(err) => Left(c2e(Coproduct[ServerError :+: Err :+: CNil](err)))
case Right(value) => Right(outToResponse(value))
}.recover(t => Left(c2e(Coproduct[ServerError :+: Err :+: CNil](toServerError(t)))))
}
/** Like recoverServerErrorEither but routes IllegalArgumentException to BadRequest.
* Use for endpoints where BadRequest is in the error coproduct and user input errors
* (like require() failures) should return 400 instead of 500.
*
* - Wraps server logic in tapir endpoints for catching any exception.
* - IllegalArgumentException is lifted to BadRequest (400 code).
* - Other exceptions are lifted to ServerError (500 code).
* - Used when the input error type, `Err`, is itself a Coproduct that contains BadRequest (BadRequest :+: NotFound :+: CNil).
* - The Left of the output Either will itself be a nested either with all coproduct elements accounted for.
* This is used for tapir endpoint definition as the errorOut type.
*/
def recoverServerErrorEitherWithUserError[In, Out, Err <: Coproduct](
fa: Future[Either[Err, In]],
)(outToResponse: In => Out)(implicit
basisErr: Basis[ServerError :+: Err, Err],
injectBadRequest: Inject[Err, BadRequest],
c2e: CoproductToEither[ServerError :+: Err],
): Future[Either[c2e.Out, Out]] = {
implicit val ec: ExecutionContext = ExecutionContext.parasitic
fa.map {
case Left(err) => Left(c2e(err.embed[ServerError :+: Err]))
case Right(value) => Right(outToResponse(value))
}.recover {
case iae: IllegalArgumentException =>
val badReq = injectBadRequest(toBadRequest(iae))
Left(c2e(badReq.embed[ServerError :+: Err]))
case t =>
Left(c2e(Coproduct[ServerError :+: Err](toServerError(t))))
}
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/YamlCodec.scala
================================================
package com.thatdot.api.v2
import io.circe._
import io.circe.syntax._
import sttp.model.MediaType
import sttp.tapir.{Codec, CodecFormat, DecodeResult, Schema}
case class YamlCodecFormat() extends CodecFormat {
override val mediaType: MediaType = MediaType("application", "yaml")
}
object YamlCodec {
def createCodec[T]()(implicit
tSchema: Schema[T],
encoder: Encoder[T],
decoder: Decoder[T],
): Codec[String, T, YamlCodecFormat] =
new Codec[String, T, YamlCodecFormat] {
override def rawDecode(s: String): DecodeResult[T] =
yaml.parser.parse(s).flatMap(_.as[T]) match {
case Left(fail: Error) => DecodeResult.Error(s, fail)
case Right(t) => DecodeResult.Value[T](t)
}
override def encode(h: T): String = yaml.Printer(dropNullKeys = true).pretty(h.asJson)
override def schema: Schema[T] = tSchema
override def format = YamlCodecFormat()
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/codec/DisjointEither.scala
================================================
package com.thatdot.api.v2.codec
import io.circe.{Decoder, Encoder}
/** Evidence that A and B are structurally disjoint in JSON (one is primitive,
* one is object, etc.) enabling unambiguous Either encoding without a wrapper object.
*
* When two types serialize to structurally distinguishable JSON (e.g., a string vs
* an object), we can encode `Either[A, B]` directly as either A's or B's JSON
* representation, and decode by attempting A first, then B.
*
* Usage:
* {{{
* import com.thatdot.api.v2.codec.DisjointEither.syntax._
* import com.thatdot.api.v2.codec.DisjointEvidence.JsonObjLike
*
* // Mark your case class as object-like
* implicit val myTypeObjLike: JsonObjLike[MyType] = new JsonObjLike[MyType] {}
*
* // Now Either[String, MyType] has encoder/decoder automatically
* val codec: Encoder[Either[String, MyType]] = implicitly
* }}}
*/
sealed trait DisjointEvidence[A, B]
object DisjointEvidence {
/** Marker for JSON primitive types (String, Int, Boolean, etc.) */
trait JsonPrim[A]
/** Marker for JSON array-like types (List, Set, etc.) */
trait JsonListLike[A]
/** Marker for JSON object-like types (case classes, Map, etc.) */
trait JsonObjLike[A]
// Built-in JsonPrim instances
implicit val jsonPrimInt: JsonPrim[Int] = new JsonPrim[Int] {}
implicit val jsonPrimString: JsonPrim[String] = new JsonPrim[String] {}
implicit val jsonPrimBoolean: JsonPrim[Boolean] = new JsonPrim[Boolean] {}
// Built-in JsonListLike instances
implicit def jsonListLikeList[A]: JsonListLike[List[A]] = new JsonListLike[List[A]] {}
implicit def jsonListLikeSet[A]: JsonListLike[Set[A]] = new JsonListLike[Set[A]] {}
// Built-in JsonObjLike instances
implicit def jsonObjLikeMap[K, V]: JsonObjLike[Map[K, V]] = new JsonObjLike[Map[K, V]] {}
// Disjoint evidence derivations (6 combinations of Prim/List/Obj)
implicit def primObj[A: JsonPrim, B: JsonObjLike]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
implicit def objPrim[A: JsonObjLike, B: JsonPrim]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
implicit def primList[A: JsonPrim, B: JsonListLike]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
implicit def listPrim[A: JsonListLike, B: JsonPrim]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
implicit def listObj[A: JsonListLike, B: JsonObjLike]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
implicit def objList[A: JsonObjLike, B: JsonListLike]: DisjointEvidence[A, B] = new DisjointEvidence[A, B] {}
}
/** Provides Either codecs when disjointness evidence exists.
*
* Mix in `DisjointEitherOps` or import `DisjointEither.syntax._` to get
* implicit `Encoder[Either[A, B]]` and `Decoder[Either[A, B]]` when
* `DisjointEvidence[A, B]` is available.
*/
object DisjointEither {
object syntax extends DisjointEitherOps
}
trait DisjointEitherOps {
implicit def disjointEitherEncoder[A, B](implicit
ev: DisjointEvidence[A, B],
encodeA: Encoder[A],
encodeB: Encoder[B],
): Encoder[Either[A, B]] = {
case Left(value) => encodeA(value)
case Right(value) => encodeB(value)
}
implicit def disjointEitherDecoder[A, B](implicit
ev: DisjointEvidence[A, B],
decodeA: Decoder[A],
decodeB: Decoder[B],
): Decoder[Either[A, B]] =
decodeA.map(Left(_)).or(decodeB.map(Right(_)))
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/codec/ThirdPartyCodecs.scala
================================================
package com.thatdot.api.v2.codec
import java.nio.charset.Charset
import java.time.Instant
import scala.util.Try
import io.circe.{Decoder, Encoder}
/** Circe codecs for third-party types that cannot have implicits in their companion objects.
*
* Usage:
* {{{
* import com.thatdot.api.v2.codec.ThirdPartyCodecs.jdk._
* }}}
*
* @see [[com.thatdot.api.v2.schema.ThirdPartySchemas]] for Tapir schemas (OpenAPI documentation)
*/
object ThirdPartyCodecs {
/** Circe codecs for JDK types */
object jdk {
implicit val charsetEncoder: Encoder[Charset] = Encoder.encodeString.contramap(_.name)
implicit val charsetDecoder: Decoder[Charset] = Decoder.decodeString.map(s => Charset.forName(s))
implicit val instantEncoder: Encoder[Instant] = Encoder.encodeString.contramap(_.toString)
implicit val instantDecoder: Decoder[Instant] = Decoder.decodeString.emapTry(s => Try(Instant.parse(s)))
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/outputs/DestinationSteps.scala
================================================
package com.thatdot.api.v2.outputs
import io.circe.{Decoder, Encoder}
import com.thatdot.api.v2.{AwsCredentials, AwsRegion, SaslJaasConfig}
import com.thatdot.common.security.Secret
/** The ADT for shared result destinations. These correspond to the API types in each product, but only exist
* for more convenient lowering to an interpreter. It's easier to automatically derive a conversion between
* structurally identical case classes than to separately write the lowering function that allocates resources
* for the interpreter.
*
* They also provide a place to define metadata for use in Tapir Schema annotations.
*/
sealed trait DestinationSteps
object DestinationSteps {
val title = "Destination Steps"
val description = "Steps that transform results on their way to a destination."
final case class Drop() extends DestinationSteps
object Drop {
val title = "Drop"
val description = "Effectively no destination at all, this does nothing but forget the data sent to it."
}
final case class File(
path: String,
) extends DestinationSteps
// with Format // Return this when prepared to support Protobuf (or more) in File writes
object File {
val propertyEncodedExampleForPath = "/temp/results.out"
val description: String = """Writes each result as a single-line JSON record.
|For the format of the result, see "Standing Query Result Output".""".stripMargin
val title = "Write JSON to File"
}
final case class HttpEndpoint(
url: String,
parallelism: Int = HttpEndpoint.propertyDefaultValueForParallelism,
headers: Map[String, Secret] = Map.empty,
) extends DestinationSteps
object HttpEndpoint {
val propertyEncodedExampleForUrl = "https://results.example.com/result-type"
val propertyDefaultValueForParallelism = 8
val propertyDefaultValueForHeaders: Map[String, Secret] = Map.empty
val propertyDescriptionForHeaders =
"Additional HTTP headers to include in the request. Header values are redacted in API responses."
val description =
"Makes an HTTP[S] POST for each result. For the format of the result, see \"Standing Query Result Output\"."
val title = "POST to HTTP[S] Webhook"
}
case class KafkaPropertyValue(s: String) extends AnyVal
object KafkaPropertyValue {
import io.circe.syntax.EncoderOps
import sttp.tapir.Schema
implicit val encoder: Encoder[KafkaPropertyValue] = Encoder.encodeString.contramap(_.s)
implicit val decoder: Decoder[KafkaPropertyValue] = Decoder.decodeString.map(KafkaPropertyValue.apply)
implicit val schema: Schema[KafkaPropertyValue] = Schema.string[KafkaPropertyValue]
private val exampleKafkaProperties: Map[String, KafkaPropertyValue] = Map(
"security.protocol" -> KafkaPropertyValue("SSL"),
"ssl.keystore.type" -> KafkaPropertyValue("PEM"),
"ssl.keystore.certificate.chain" -> KafkaPropertyValue("/path/to/file/containing/certificate/chain"),
"ssl.key.password" -> KafkaPropertyValue("private_key_password"),
"ssl.truststore.type" -> KafkaPropertyValue("PEM"),
"ssl.truststore.certificates" -> KafkaPropertyValue("/path/to/truststore/certificate"),
)
implicit lazy val mapSchema: Schema[Map[String, KafkaPropertyValue]] =
Schema
.schemaForMap[KafkaPropertyValue](schema)
.encodedExample(exampleKafkaProperties.asJson)
}
final case class Kafka(
topic: String,
bootstrapServers: String,
format: OutputFormat = Kafka.propertyDefaultValueForFormat,
sslKeystorePassword: Option[Secret] = None,
sslTruststorePassword: Option[Secret] = None,
sslKeyPassword: Option[Secret] = None,
saslJaasConfig: Option[SaslJaasConfig] = None,
kafkaProperties: Map[String, KafkaPropertyValue] = Kafka.propertyDefaultValueForKafkaProperties,
) extends DestinationSteps
with Format
object Kafka {
val propertyEncodedExampleForBootstrapServers = "kafka.svc.cluster.local:9092"
val propertyEncodedExampleForTopic = "example-topic"
val propertyDefaultValueForFormat: OutputFormat = OutputFormat.JSON
val propertyDefaultValueForKafkaProperties: Map[String, KafkaPropertyValue] = Map.empty
val propertyDefaultValueEncodedForKafkaProperties: Some[String] = Some("{}")
val propertyDescriptionForKafkaProperties: String = """Map of Kafka producer properties.
|See <https://kafka.apache.org/documentation.html#producerconfigs>""".stripMargin
val description = "Publishes provided data to the specified Apache Kafka topic."
val title = "Publish to Kafka Topic"
}
final case class Kinesis(
credentials: Option[AwsCredentials],
region: Option[AwsRegion],
streamName: String,
format: OutputFormat = Kinesis.propertyDefaultValueForFormat,
kinesisParallelism: Option[Int],
kinesisMaxBatchSize: Option[Int],
kinesisMaxRecordsPerSecond: Option[Int],
kinesisMaxBytesPerSecond: Option[Int],
) extends DestinationSteps
with Format
object Kinesis {
val propertyEncodedExampleForStreamName = "example-stream"
val propertyDefaultValueForFormat: OutputFormat = OutputFormat.JSON
val description = "Publishes provided data to the specified Amazon Kinesis stream."
val title = "Publish to Kinesis Data Stream"
}
final case class ReactiveStream(
address: String = ReactiveStream.propertyDefaultValueForAddress,
port: Int,
format: OutputFormat,
) extends DestinationSteps
with Format
object ReactiveStream {
val propertyDescriptionForAddress = "The address to bind the reactive stream server on."
val propertyDefaultValueForAddress = "localhost"
val propertyDescriptionForPort = "The port to bind the reactive stream server on."
val description: String =
"""Broadcasts data to a created Reactive Stream. Other thatDot products can subscribe to Reactive Streams.
|⚠️ Warning: Reactive Stream outputs do not function correctly when running in a cluster.""".stripMargin
val title = "Broadcast to Reactive Stream"
}
final case class SNS(
credentials: Option[AwsCredentials],
region: Option[AwsRegion],
topic: String,
format: OutputFormat,
) extends DestinationSteps
with Format
object SNS {
val propertyEncodedExampleForTopic = "example-topic"
val propertyDescriptionForTopic = "ARN of the topic to publish to."
val description: String = """Publishes an AWS SNS record to the provided topic.
|⚠️ <b><em>Double check your credentials and topic ARN!</em></b> If writing to SNS fails, the write will
|be retried indefinitely. If the error is unfixable (e.g., the topic or credentials
|cannot be found), the outputs will never be emitted and the Standing Query this output
|is attached to may stop running.""".stripMargin // Use StringOps#asOneLine when that is accessible
val title = "Publish to SNS Topic"
}
final case class StandardOut() extends DestinationSteps
object StandardOut {
val description = "Prints each result as a single-line JSON object to stdout on the application server."
val title = "Log JSON to Console"
}
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/outputs/Format.scala
================================================
package com.thatdot.api.v2.outputs
trait Format {
val format: OutputFormat
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/outputs/OutputFormat.scala
================================================
package com.thatdot.api.v2.outputs
import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder}
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.{description, encodedExample, title}
import com.thatdot.api.v2.TypeDiscriminatorConfig.instances.circeConfig
@title("Result Output Format")
sealed trait OutputFormat
object OutputFormat {
implicit lazy val protobufSchema: Schema[Protobuf] = Schema.derived
implicit lazy val schema: Schema[OutputFormat] = Schema.derived
@title("JSON")
@encodedExample("JSON")
case object JSON extends OutputFormat
@title("Protobuf")
@encodedExample("""{
| "type": "Protobuf",
| "schemaUrl": "conf/protobuf-schemas/example_schema.desc",
| "typeName": "ExampleType"
|}""".stripMargin)
final case class Protobuf(
@description(
"URL (or local filename) of the Protobuf .desc file to load that contains the desired typeName to serialize to",
)
@encodedExample("conf/protobuf-schemas/example_schema.desc")
schemaUrl: String,
@description("message type name to use (from the given .desc file) as the message type")
@encodedExample("ExampleType")
typeName: String,
) extends OutputFormat
implicit val encoder: Encoder[OutputFormat] = deriveConfiguredEncoder
implicit val decoder: Decoder[OutputFormat] = deriveConfiguredDecoder
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/schema/TapirJsonConfig.scala
================================================
package com.thatdot.api.v2.schema
import io.circe.Printer
import sttp.tapir.json.circe.TapirJsonCirce
/** Provides `jsonBody[T]` for endpoint definitions, using overridden settings. */
trait TapirJsonConfig extends TapirJsonCirce {
override def jsonPrinter: Printer = TapirJsonConfig.printer
}
object TapirJsonConfig extends TapirJsonConfig {
/** Circe JSON printer that will
* - Drop null values from output JSON
* - Use no indentation (compact output)
*/
private val printer: Printer = Printer(dropNullValues = true, indent = "")
}
================================================
FILE: api/src/main/scala/com/thatdot/api/v2/schema/ThirdPartySchemas.scala
================================================
package com.thatdot.api.v2.schema
import java.nio.charset.Charset
import java.time.Instant
import scala.util.{Failure, Success, Try}
import cats.data.NonEmptyList
import io.circe.Json
import sttp.tapir.CodecFormat.TextPlain
import sttp.tapir.{Codec, DecodeResult, Schema}
/** Tapir schemas for third-party types that cannot have implicits in their companion objects.
*
* Usage:
* {{{
* import com.thatdot.api.v2.schema.ThirdPartySchemas.cats._
* import com.thatdot.api.v2.schema.ThirdPartySchemas.circe._
* import com.thatdot.api.v2.schema.ThirdPartySchemas.jdk._
* }}}
*
* @see [[com.thatdot.api.v2.codec.ThirdPartyCodecs]] for Circe codecs (JSON serialization)
*/
object ThirdPartySchemas {
/** Schemas for `cats` data types */
object cats {
implicit def nonEmptyListSchema[A](implicit inner: Schema[A]): Schema[NonEmptyList[A]] =
Schema.schemaForIterable[A, List].map(list => NonEmptyList.fromList(list))(_.toList)
}
/** Schemas for Circe types */
object circe {
implicit lazy val jsonSchema: Schema[Json] = Schema.any[Json]
implicit lazy val mapStringJsonSchema: Schema[Map[String, Json]] = Schema.schemaForMap[String, Json](identity)
implicit lazy val seqJsonSchema: Schema[Seq[Json]] = jsonSchema.asIterable[Seq]
implicit lazy val seqSeqJsonSchema: Schema[Seq[Seq[Json]]] = seqJsonSchema.asIterable[Seq]
}
/** Schemas for JDK types */
object jdk {
implicit val charsetCodec: Codec[String, Charset, TextPlain] = Codec.string.mapDecode(s =>
Try(Charset.forName(s)) match {
case Success(charset) => DecodeResult.Value(charset)
case Failure(e) => DecodeResult.Error(s"Invalid charset: $s", e)
},
)(_.toString)
implicit lazy val charsetSchema: Schema[Charset] = charsetCodec.schema
implicit val instantCodec: Codec[String, Instant, TextPlain] = Codec.string.mapDecode(s =>
Try(Instant.parse(s)) match {
case Success(instant) => DecodeResult.Value(instant)
case Failure(e) => DecodeResult.Error(s"Invalid instant: $s", e)
},
)(_.toString)
implicit lazy val instantSchema: Schema[Instant] = instantCodec.schema
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/codec/SecretCodecsSpec.scala
================================================
package com.thatdot.api.codec
import io.circe.Json
import io.circe.syntax.EncoderOps
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import com.thatdot.api.codec.SecretCodecs._
import com.thatdot.common.security.Secret
class SecretCodecsSpec extends AnyWordSpec with Matchers {
"secretEncoder" should {
"redact the actual value" in {
val secret = Secret("AKIAIOSFODNN7EXAMPLE")
secret.asJson shouldBe Json.fromString("Secret(****)")
}
}
"secretDecoder" should {
"wrap incoming string and preserve value internally" in {
import Secret.Unsafe._
val originalValue = "my-secret-value"
val json = Json.fromString(originalValue)
val decoded = json.as[Secret].getOrElse(fail("Failed to decode Secret"))
decoded.toString shouldBe "Secret(****)"
decoded.unsafeValue shouldBe originalValue
}
}
"preservingEncoder" should {
"preserve actual credential value" in {
import Secret.Unsafe._
val value = "real-credential-value"
Secret(value).asJson(preservingEncoder) shouldBe Json.fromString(value)
}
"produce different output than standard encoder" in {
import Secret.Unsafe._
val secret = Secret("credential")
secret.asJson(secretEncoder) shouldNot be(secret.asJson(preservingEncoder))
}
"preserve value through roundtrip" in {
import Secret.Unsafe._
val originalValue = "AKIAIOSFODNN7EXAMPLE"
val secret = Secret(originalValue)
val json = secret.asJson(preservingEncoder)
val decoded = json.as[Secret].getOrElse(fail("Failed to decode Secret"))
decoded.unsafeValue shouldBe originalValue
}
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/ApiErrorsCodecSpec.scala
================================================
package com.thatdot.api.v2
import io.circe.syntax.EncoderOps
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
class ApiErrorsCodecSpec extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyChecks {
import ErrorTypeGenerators.Arbs._
import ErrorResponseGenerators.Arbs._
test("ErrorType.ApiError encodes message field") {
forAll { (error: ErrorType) =>
val json = error.asJson
json.hcursor.get[String]("message") shouldBe Right(error.message)
}
}
test("ErrorType.DecodeError encodes optional help field") {
forAll { (error: ErrorType.DecodeError) =>
val json = error.asJson
error.help match {
case Some(h) => json.hcursor.get[String]("help") shouldBe Right(h)
case None => json.hcursor.get[String]("help").isLeft shouldBe true
}
}
}
test("ErrorType encodes with type discriminator") {
(ErrorType.ApiError("msg"): ErrorType).asJson.hcursor.get[String]("type") shouldBe Right("ApiError")
(ErrorType.DecodeError("msg"): ErrorType).asJson.hcursor.get[String]("type") shouldBe Right("DecodeError")
(ErrorType.CypherError("msg"): ErrorType).asJson.hcursor.get[String]("type") shouldBe Right("CypherError")
}
test("ErrorResponse.ServerError encodes errors list") {
forAll { (error: ErrorResponse.ServerError) =>
val json = error.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray)
errorsArray.isDefined shouldBe true
errorsArray.get.size shouldBe error.errors.size
}
}
test("ErrorResponse.BadRequest encodes errors list") {
forAll { (error: ErrorResponse.BadRequest) =>
val json = error.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray)
errorsArray.get.size shouldBe error.errors.size
}
}
test("ErrorResponse.NotFound encodes errors list") {
forAll { (error: ErrorResponse.NotFound) =>
val json = error.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray)
errorsArray.get.size shouldBe error.errors.size
}
}
test("ErrorResponse.Unauthorized encodes errors list") {
forAll { (error: ErrorResponse.Unauthorized) =>
val json = error.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray)
errorsArray.get.size shouldBe error.errors.size
}
}
test("ErrorResponse.ServiceUnavailable encodes errors list") {
forAll { (error: ErrorResponse.ServiceUnavailable) =>
val json = error.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray)
errorsArray.get.size shouldBe error.errors.size
}
}
test("ErrorResponse types preserve error content when encoded") {
val errorList = List(ErrorType.ApiError("error1"), ErrorType.CypherError("error2"))
val serverError = ErrorResponse.ServerError(errorList)
val json = serverError.asJson
val errorsArray = json.hcursor.downField("errors").focus.flatMap(_.asArray).get
errorsArray.size shouldBe 2
errorsArray.head.hcursor.get[String]("message") shouldBe Right("error1")
errorsArray.head.hcursor.get[String]("type") shouldBe Right("ApiError")
errorsArray(1).hcursor.get[String]("message") shouldBe Right("error2")
errorsArray(1).hcursor.get[String]("type") shouldBe Right("CypherError")
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/AwsCredentialsCodecSpec.scala
================================================
package com.thatdot.api.v2
import io.circe.Json
import io.circe.syntax.EncoderOps
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
import com.thatdot.common.security.Secret
class AwsCredentialsCodecSpec extends AnyWordSpec with Matchers with ScalaCheckDrivenPropertyChecks {
import AwsGenerators.Arbs._
"AwsCredentials encoder" should {
"redact credentials in JSON output" in {
val creds = AwsCredentials(
accessKeyId = Secret("AKIAIOSFODNN7EXAMPLE"),
secretAccessKey = Secret("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"),
)
creds.asJson shouldBe Json.obj(
"accessKeyId" -> Json.fromString("Secret(****)"),
"secretAccessKey" -> Json.fromString("Secret(****)"),
)
}
}
"AwsCredentials decoder" should {
"decode JSON with plain strings" in {
import Secret.Unsafe._
val json = Json.obj(
"accessKeyId" -> Json.fromString("AKIATEST"),
"secretAccessKey" -> Json.fromString("secretkey123"),
)
val creds = json.as[AwsCredentials].getOrElse(fail("Failed to decode AwsCredentials"))
creds.accessKeyId.unsafeValue shouldBe "AKIATEST"
creds.secretAccessKey.unsafeValue shouldBe "secretkey123"
}
"decode values correctly for any credentials (property-based)" in {
import Secret.Unsafe._
forAll { (creds: AwsCredentials) =>
val originalAccessKey = creds.accessKeyId.unsafeValue
val originalSecretKey = creds.secretAccessKey.unsafeValue
val inputJson = Json.obj(
"accessKeyId" -> Json.fromString(originalAccessKey),
"secretAccessKey" -> Json.fromString(originalSecretKey),
)
val decoded = inputJson.as[AwsCredentials].getOrElse(fail("Failed to decode AwsCredentials"))
decoded.accessKeyId.unsafeValue shouldBe originalAccessKey
decoded.secretAccessKey.unsafeValue shouldBe originalSecretKey
}
}
}
"AwsCredentials.preservingEncoder" should {
"preserve credential values in JSON output" in {
import Secret.Unsafe._
val accessKey = "AKIAIOSFODNN8EXAMPLE"
val secretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
val creds = AwsCredentials(
accessKeyId = Secret(accessKey),
secretAccessKey = Secret(secretKey),
)
val json = creds.asJson(AwsCredentials.preservingEncoder)
json shouldBe Json.obj(
"accessKeyId" -> Json.fromString(accessKey),
"secretAccessKey" -> Json.fromString(secretKey),
)
}
"produce different output than standard encoder" in {
import Secret.Unsafe._
val accessKey = "AKIA123"
val secretKey = "secret456"
val creds = AwsCredentials(
accessKeyId = Secret(accessKey),
secretAccessKey = Secret(secretKey),
)
val redacted = creds.asJson
val preserved = creds.asJson(AwsCredentials.preservingEncoder)
redacted.hcursor.downField("accessKeyId") shouldNot be(preserved.hcursor.downField("accessKeyId"))
redacted.hcursor.downField("secretAccessKey") shouldNot be(preserved.hcursor.downField("secretAccessKey"))
}
"preserve values through roundtrip (property-based)" in {
import Secret.Unsafe._
forAll { (creds: AwsCredentials) =>
val originalAccessKey = creds.accessKeyId.unsafeValue
val originalSecretKey = creds.secretAccessKey.unsafeValue
val json = creds.asJson(AwsCredentials.preservingEncoder)
val decoded = json.as[AwsCredentials].getOrElse(fail("Failed to decode AwsCredentials"))
decoded.accessKeyId.unsafeValue shouldBe originalAccessKey
decoded.secretAccessKey.unsafeValue shouldBe originalSecretKey
}
}
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/AwsGenerators.scala
================================================
package com.thatdot.api.v2
import org.scalacheck.{Arbitrary, Gen}
import com.thatdot.common.security.Secret
import com.thatdot.quine.ScalaPrimitiveGenerators
object AwsGenerators {
import ScalaPrimitiveGenerators.Gens.nonEmptyAlphaNumStr
object Gens {
val awsCredentials: Gen[AwsCredentials] = for {
accessKey <- nonEmptyAlphaNumStr
secretKey <- nonEmptyAlphaNumStr
} yield AwsCredentials(Secret(accessKey), Secret(secretKey))
val optAwsCredentials: Gen[Option[AwsCredentials]] = Gen.option(awsCredentials)
val awsRegion: Gen[AwsRegion] =
Gen.oneOf("us-east-1", "us-west-2", "eu-west-1", "ap-northeast-1").map(AwsRegion.apply)
val optAwsRegion: Gen[Option[AwsRegion]] = Gen.option(awsRegion)
}
object Arbs {
implicit val arbAwsCredentials: Arbitrary[AwsCredentials] = Arbitrary(Gens.awsCredentials)
implicit val arbOptAwsCredentials: Arbitrary[Option[AwsCredentials]] = Arbitrary(Gens.optAwsCredentials)
implicit val arbAwsRegion: Arbitrary[AwsRegion] = Arbitrary(Gens.awsRegion)
implicit val arbOptAwsRegion: Arbitrary[Option[AwsRegion]] = Arbitrary(Gens.optAwsRegion)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/AwsRegionCodecSpec.scala
================================================
package com.thatdot.api.v2
import io.circe.Json
import io.circe.syntax.EncoderOps
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
class AwsRegionCodecSpec extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyChecks {
import AwsGenerators.Arbs._
test("AwsRegion encodes as plain string") {
val region = AwsRegion("us-west-2")
region.asJson shouldBe Json.fromString("us-west-2")
}
test("AwsRegion decodes from plain string") {
val json = Json.fromString("us-west-2")
json.as[AwsRegion] shouldBe Right(AwsRegion("us-west-2"))
}
test("AwsRegion roundtrips encode/decode") {
forAll { (region: AwsRegion) =>
region.asJson.as[AwsRegion] shouldBe Right(region)
}
}
test("Option[AwsRegion] roundtrips encode/decode") {
forAll { (region: Option[AwsRegion]) =>
region.asJson.as[Option[AwsRegion]] shouldBe Right(region)
}
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/ErrorResponseGenerators.scala
================================================
package com.thatdot.api.v2
import org.scalacheck.{Arbitrary, Gen}
import com.thatdot.quine.ScalaPrimitiveGenerators
object ErrorResponseGenerators {
import ScalaPrimitiveGenerators.Gens.smallPosNum
import ErrorTypeGenerators.Gens.errorType
object Gens {
val errorList: Gen[List[ErrorType]] = smallPosNum.flatMap(Gen.listOfN(_, errorType))
val serverError: Gen[ErrorResponse.ServerError] = errorList.map(ErrorResponse.ServerError(_))
val badRequest: Gen[ErrorResponse.BadRequest] = errorList.map(ErrorResponse.BadRequest(_))
val notFound: Gen[ErrorResponse.NotFound] = errorList.map(ErrorResponse.NotFound(_))
val unauthorized: Gen[ErrorResponse.Unauthorized] = errorList.map(ErrorResponse.Unauthorized(_))
val serviceUnavailable: Gen[ErrorResponse.ServiceUnavailable] = errorList.map(ErrorResponse.ServiceUnavailable(_))
}
object Arbs {
implicit val serverError: Arbitrary[ErrorResponse.ServerError] = Arbitrary(Gens.serverError)
implicit val badRequest: Arbitrary[ErrorResponse.BadRequest] = Arbitrary(Gens.badRequest)
implicit val notFound: Arbitrary[ErrorResponse.NotFound] = Arbitrary(Gens.notFound)
implicit val unauthorized: Arbitrary[ErrorResponse.Unauthorized] = Arbitrary(Gens.unauthorized)
implicit val serviceUnavailable: Arbitrary[ErrorResponse.ServiceUnavailable] = Arbitrary(Gens.serviceUnavailable)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/ErrorTypeGenerators.scala
================================================
package com.thatdot.api.v2
import org.scalacheck.{Arbitrary, Gen}
import com.thatdot.quine.ScalaPrimitiveGenerators
object ErrorTypeGenerators {
import ScalaPrimitiveGenerators.Gens.{nonEmptyAlphaNumStr, optNonEmptyAlphaNumStr}
object Gens {
val apiError: Gen[ErrorType.ApiError] =
nonEmptyAlphaNumStr.map(ErrorType.ApiError(_))
val decodeError: Gen[ErrorType.DecodeError] = for {
message <- nonEmptyAlphaNumStr
help <- optNonEmptyAlphaNumStr
} yield ErrorType.DecodeError(message, help)
val cypherError: Gen[ErrorType.CypherError] =
nonEmptyAlphaNumStr.map(ErrorType.CypherError(_))
val errorType: Gen[ErrorType] =
Gen.oneOf(apiError, decodeError, cypherError)
}
object Arbs {
implicit val apiError: Arbitrary[ErrorType.ApiError] = Arbitrary(Gens.apiError)
implicit val decodeError: Arbitrary[ErrorType.DecodeError] = Arbitrary(Gens.decodeError)
implicit val cypherError: Arbitrary[ErrorType.CypherError] = Arbitrary(Gens.cypherError)
implicit val errorType: Arbitrary[ErrorType] = Arbitrary(Gens.errorType)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigCodecSpec.scala
================================================
package com.thatdot.api.v2
import io.circe.syntax.EncoderOps
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import com.thatdot.common.security.Secret
/** Tests for [[SaslJaasConfig]] Circe codec behavior.
*
* Verifies that:
* - Secret fields (password, clientSecret) are redacted in JSON output
* - Non-sensitive fields (username, clientId) are NOT redacted
* - Decoder can reconstruct case classes from JSON
*/
class SaslJaasConfigCodecSpec extends AnyFunSuite with Matchers {
test("PlainLogin encoder redacts password") {
val login = PlainLogin(username = "alice", password = Secret("test-pw"))
val json = login.asJson
json.hcursor.get[String]("password") shouldBe Right("Secret(****)")
}
test("PlainLogin encoder does NOT redact username") {
val login = PlainLogin(username = "alice", password = Secret("test-pw"))
val json = login.asJson
json.hcursor.get[String]("username") shouldBe Right("alice")
}
test("PlainLogin decoder reconstructs from JSON") {
import Secret.Unsafe._
val json = io.circe.parser
.parse("""{"username": "alice", "password": "test-pw"}""")
.getOrElse(fail("Failed to parse JSON"))
val decoded = json.as[PlainLogin].getOrElse(fail("Failed to decode PlainLogin"))
decoded.username shouldBe "alice"
decoded.password.unsafeValue shouldBe "test-pw"
}
test("ScramLogin encoder redacts password") {
val login = ScramLogin(username = "bob", password = Secret("secret123"))
val json = login.asJson
json.hcursor.get[String]("password") shouldBe Right("Secret(****)")
}
test("ScramLogin encoder does NOT redact username") {
val login = ScramLogin(username = "bob", password = Secret("secret123"))
val json = login.asJson
json.hcursor.get[String]("username") shouldBe Right("bob")
}
test("ScramLogin decoder reconstructs from JSON") {
import Secret.Unsafe._
val json = io.circe.parser
.parse("""{"username": "bob", "password": "secret123"}""")
.getOrElse(fail("Failed to parse JSON"))
val decoded = json.as[ScramLogin].getOrElse(fail("Failed to decode ScramLogin"))
decoded.username shouldBe "bob"
decoded.password.unsafeValue shouldBe "secret123"
}
test("OAuthBearerLogin encoder redacts clientSecret") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data"),
tokenEndpointUrl = Some("https://auth.example.com/token"),
)
val json = login.asJson
json.hcursor.get[String]("clientSecret") shouldBe Right("Secret(****)")
}
test("OAuthBearerLogin encoder does NOT redact clientId") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data"),
tokenEndpointUrl = Some("https://auth.example.com/token"),
)
val json = login.asJson
json.hcursor.get[String]("clientId") shouldBe Right("my-client")
}
test("OAuthBearerLogin encoder does NOT redact scope") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data"),
tokenEndpointUrl = None,
)
val json = login.asJson
json.hcursor.get[Option[String]]("scope") shouldBe Right(Some("read:data"))
}
test("OAuthBearerLogin encoder does NOT redact tokenEndpointUrl") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = None,
tokenEndpointUrl = Some("https://auth.example.com/token"),
)
val json = login.asJson
json.hcursor.get[Option[String]]("tokenEndpointUrl") shouldBe Right(Some("https://auth.example.com/token"))
}
test("OAuthBearerLogin decoder reconstructs from JSON with all fields") {
import Secret.Unsafe._
val json = io.circe.parser
.parse(
"""{"clientId": "my-client", "clientSecret": "oauth-secret", "scope": "read:data", "tokenEndpointUrl": "https://auth.example.com/token"}""",
)
.getOrElse(fail("Failed to parse JSON"))
val decoded = json.as[OAuthBearerLogin].getOrElse(fail("Failed to decode OAuthBearerLogin"))
decoded.clientId shouldBe "my-client"
decoded.clientSecret.unsafeValue shouldBe "oauth-secret"
decoded.scope shouldBe Some("read:data")
decoded.tokenEndpointUrl shouldBe Some("https://auth.example.com/token")
}
test("OAuthBearerLogin decoder applies defaults for optional fields") {
import Secret.Unsafe._
val json = io.circe.parser
.parse("""{"clientId": "my-client", "clientSecret": "oauth-secret"}""")
.getOrElse(fail("Failed to parse JSON"))
val decoded = json.as[OAuthBearerLogin].getOrElse(fail("Failed to decode OAuthBearerLogin"))
decoded.clientId shouldBe "my-client"
decoded.clientSecret.unsafeValue shouldBe "oauth-secret"
decoded.scope shouldBe None
decoded.tokenEndpointUrl shouldBe None
}
test("SaslJaasConfig sealed trait encodes with type discriminator") {
val plain: SaslJaasConfig = PlainLogin(username = "alice", password = Secret("pw"))
val scram: SaslJaasConfig = ScramLogin(username = "bob", password = Secret("pw"))
val oauth: SaslJaasConfig = OAuthBearerLogin(clientId = "client", clientSecret = Secret("secret"))
plain.asJson.hcursor.get[String]("type") shouldBe Right("PlainLogin")
scram.asJson.hcursor.get[String]("type") shouldBe Right("ScramLogin")
oauth.asJson.hcursor.get[String]("type") shouldBe Right("OAuthBearerLogin")
}
test("SaslJaasConfig decoder routes to correct subtype via type discriminator") {
import Secret.Unsafe._
val plainJson = io.circe.parser
.parse("""{"type": "PlainLogin", "username": "alice", "password": "pw"}""")
.getOrElse(fail("Failed to parse JSON"))
val decoded = plainJson.as[SaslJaasConfig].getOrElse(fail("Failed to decode SaslJaasConfig"))
decoded shouldBe a[PlainLogin]
val plain = decoded.asInstanceOf[PlainLogin]
plain.username shouldBe "alice"
plain.password.unsafeValue shouldBe "pw"
}
test("toJaasConfigString produces PlainLoginModule JAAS string for PlainLogin") {
import Secret.Unsafe._
val login = PlainLogin(username = "alice", password = Secret("my-password"))
val jaasString = SaslJaasConfig.toJaasConfigString(login)
jaasString shouldBe """org.apache.kafka.common.security.plain.PlainLoginModule required username="alice" password="my-password";"""
}
test("toJaasConfigString produces ScramLoginModule JAAS string for ScramLogin") {
import Secret.Unsafe._
val login = ScramLogin(username = "bob", password = Secret("scram-secret"))
val jaasString = SaslJaasConfig.toJaasConfigString(login)
jaasString shouldBe """org.apache.kafka.common.security.scram.ScramLoginModule required username="bob" password="scram-secret";"""
}
test("toJaasConfigString produces OAuthBearerLoginModule JAAS string for OAuthBearerLogin") {
import Secret.Unsafe._
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
)
val jaasString = SaslJaasConfig.toJaasConfigString(login)
jaasString should include("org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required")
jaasString should include("""clientId="my-client"""")
jaasString should include("""clientSecret="oauth-secret"""")
jaasString should endWith(";")
}
test("toJaasConfigString includes scope in OAuthBearerLogin JAAS string when present") {
import Secret.Unsafe._
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data write:data"),
)
val jaasString = SaslJaasConfig.toJaasConfigString(login)
jaasString should include("""scope="read:data write:data"""")
}
test("toJaasConfigString includes tokenEndpointUrl in OAuthBearerLogin JAAS string when present") {
import Secret.Unsafe._
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
tokenEndpointUrl = Some("https://auth.example.com/token"),
)
val jaasString = SaslJaasConfig.toJaasConfigString(login)
jaasString should include("""sasl.oauthbearer.token.endpoint.url="https://auth.example.com/token"""")
}
test("preservingEncoder preserves PlainLogin password") {
import Secret.Unsafe._
val login: SaslJaasConfig = PlainLogin(username = "alice", password = Secret("real-password"))
val encoder = SaslJaasConfig.preservingEncoder
val json = encoder(login)
json.hcursor.get[String]("password") shouldBe Right("real-password")
json.hcursor.get[String]("username") shouldBe Right("alice")
}
test("preservingEncoder preserves ScramLogin password") {
import Secret.Unsafe._
val login: SaslJaasConfig = ScramLogin(username = "bob", password = Secret("scram-secret"))
val encoder = SaslJaasConfig.preservingEncoder
val json = encoder(login)
json.hcursor.get[String]("password") shouldBe Right("scram-secret")
json.hcursor.get[String]("username") shouldBe Right("bob")
}
test("preservingEncoder preserves OAuthBearerLogin clientSecret") {
import Secret.Unsafe._
val login: SaslJaasConfig = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data"),
)
val encoder = SaslJaasConfig.preservingEncoder
val json = encoder(login)
json.hcursor.get[String]("clientSecret") shouldBe Right("oauth-secret")
json.hcursor.get[String]("clientId") shouldBe Right("my-client")
json.hcursor.get[Option[String]]("scope") shouldBe Right(Some("read:data"))
}
test("preservingEncoder includes type discriminator") {
import Secret.Unsafe._
val plain: SaslJaasConfig = PlainLogin(username = "alice", password = Secret("pw"))
val scram: SaslJaasConfig = ScramLogin(username = "bob", password = Secret("pw"))
val oauth: SaslJaasConfig = OAuthBearerLogin(clientId = "client", clientSecret = Secret("secret"))
val encoder = SaslJaasConfig.preservingEncoder
encoder(plain).hcursor.get[String]("type") shouldBe Right("PlainLogin")
encoder(scram).hcursor.get[String]("type") shouldBe Right("ScramLogin")
encoder(oauth).hcursor.get[String]("type") shouldBe Right("OAuthBearerLogin")
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigGenerators.scala
================================================
package com.thatdot.api.v2
import org.scalacheck.{Arbitrary, Gen}
import com.thatdot.common.security.Secret
import com.thatdot.quine.ScalaPrimitiveGenerators
object SaslJaasConfigGenerators {
import ScalaPrimitiveGenerators.Gens.nonEmptyAlphaNumStr
object Gens {
// This may be worth putting in into a SecretGenerators, but more likely after we pull quine-common into quine-plus
val secret: Gen[Secret] = nonEmptyAlphaNumStr.map(Secret(_))
// This may be worth putting in into a SecretGenerators, but more likely after we pull quine-common into quine-plus
val optSecret: Gen[Option[Secret]] = Gen.option(secret)
val plainLogin: Gen[PlainLogin] = for {
username <- nonEmptyAlphaNumStr
password <- secret
} yield PlainLogin(username, password)
val scramLogin: Gen[ScramLogin] = for {
username <- nonEmptyAlphaNumStr
password <- secret
} yield ScramLogin(username, password)
val oauthBearerLogin: Gen[OAuthBearerLogin] = for {
clientId <- nonEmptyAlphaNumStr
clientSecret <- secret
scope <- Gen.option(nonEmptyAlphaNumStr)
tokenEndpointUrl <- Gen.option(nonEmptyAlphaNumStr.map(s => s"https://$s.example.com/oauth/token"))
} yield OAuthBearerLogin(clientId, clientSecret, scope, tokenEndpointUrl)
val saslJaasConfig: Gen[SaslJaasConfig] =
Gen.oneOf(plainLogin, scramLogin, oauthBearerLogin)
val optSaslJaasConfig: Gen[Option[SaslJaasConfig]] = Gen.option(saslJaasConfig)
}
object Arbs {
implicit val arbSecret: Arbitrary[Secret] = Arbitrary(Gens.secret)
implicit val arbOptSecret: Arbitrary[Option[Secret]] = Arbitrary(Gens.optSecret)
implicit val arbPlainLogin: Arbitrary[PlainLogin] = Arbitrary(Gens.plainLogin)
implicit val arbScramLogin: Arbitrary[ScramLogin] = Arbitrary(Gens.scramLogin)
implicit val arbOAuthBearerLogin: Arbitrary[OAuthBearerLogin] = Arbitrary(Gens.oauthBearerLogin)
implicit val arbSaslJaasConfig: Arbitrary[SaslJaasConfig] = Arbitrary(Gens.saslJaasConfig)
implicit val arbOptSaslJaasConfig: Arbitrary[Option[SaslJaasConfig]] = Arbitrary(Gens.optSaslJaasConfig)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigLoggableSpec.scala
================================================
package com.thatdot.api.v2
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import com.thatdot.common.security.Secret
/** Tests for [[SaslJaasConfig]] Loggable instance behavior.
*
* Verifies that:
* - Sensitive fields (password, clientSecret) are redacted as "****" in logged output
* - Non-sensitive fields (username, clientId, scope, tokenEndpointUrl) are visible
* - The format matches the expected pattern for each subtype
*/
class SaslJaasConfigLoggableSpec extends AnyFunSuite with Matchers {
import SaslJaasConfig.logSaslJaasConfig
test("PlainLogin logs in JAAS format with username visible and password redacted") {
val login = PlainLogin(username = "alice", password = Secret("jaas-queen"))
val logged = logSaslJaasConfig.safe(login)
logged shouldBe """org.apache.kafka.common.security.plain.PlainLoginModule required username="alice" password="****";"""
}
test("ScramLogin logs in JAAS format with username visible and password redacted") {
val login = ScramLogin(username = "bob", password = Secret("scram-secret"))
val logged = logSaslJaasConfig.safe(login)
logged shouldBe """org.apache.kafka.common.security.scram.ScramLoginModule required username="bob" password="****";"""
}
test("OAuthBearerLogin logs in JAAS format with clientId visible and clientSecret redacted") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
scope = Some("read:data"),
tokenEndpointUrl = Some("https://auth.example.com/token"),
)
val logged = logSaslJaasConfig.safe(login)
logged shouldBe """org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required clientId="my-client" clientSecret="****" scope="read:data" sasl.oauthbearer.token.endpoint.url="https://auth.example.com/token";"""
}
test("OAuthBearerLogin logs in JAAS format without optional fields when absent") {
val login = OAuthBearerLogin(
clientId = "my-client",
clientSecret = Secret("oauth-secret"),
)
val logged = logSaslJaasConfig.safe(login)
logged shouldBe """org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required clientId="my-client" clientSecret="****";"""
}
test("PlainLogin password is indistinguishable regardless of actual value") {
val login1 = PlainLogin(username = "alice", password = Secret("password1"))
val login2 = PlainLogin(username = "alice", password = Secret("different-password"))
val logged1 = logSaslJaasConfig.safe(login1)
val logged2 = logSaslJaasConfig.safe(login2)
logged1 shouldBe logged2
}
test("ScramLogin password is indistinguishable regardless of actual value") {
val login1 = ScramLogin(username = "bob", password = Secret("password1"))
val login2 = ScramLogin(username = "bob", password = Secret("different-password"))
val logged1 = logSaslJaasConfig.safe(login1)
val logged2 = logSaslJaasConfig.safe(login2)
logged1 shouldBe logged2
}
test("OAuthBearerLogin clientSecret is indistinguishable regardless of actual value") {
val login1 = OAuthBearerLogin(clientId = "client", clientSecret = Secret("secret1"))
val login2 = OAuthBearerLogin(clientId = "client", clientSecret = Secret("different-secret"))
val logged1 = logSaslJaasConfig.safe(login1)
val logged2 = logSaslJaasConfig.safe(login2)
logged1 shouldBe logged2
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/SuccessEnvelopeCodecSpec.scala
================================================
package com.thatdot.api.v2
import io.circe.syntax.EncoderOps
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks
class SuccessEnvelopeCodecSpec extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyChecks {
import SuccessEnvelopeGenerators.Arbs._
test("SuccessEnvelope.Ok encodes content field") {
forAll { (envelope: SuccessEnvelope.Ok[String]) =>
val json = envelope.asJson
json.hcursor.get[String]("content") shouldBe Right(envelope.content)
}
}
test("SuccessEnvelope.Ok encodes optional message field") {
forAll { (envelope: SuccessEnvelope.Ok[String]) =>
val json = envelope.asJson
envelope.message match {
case Some(msg) => json.hcursor.get[String]("message") shouldBe Right(msg)
case None => json.hcursor.get[Option[String]]("message") shouldBe Right(None)
}
}
}
test("SuccessEnvelope.Ok encodes warnings list") {
forAll { (envelope: SuccessEnvelope.Ok[String]) =>
val json = envelope.asJson
json.hcursor.get[List[String]]("warnings") shouldBe Right(envelope.warnings)
}
}
test("SuccessEnvelope.Ok roundtrips encode/decode") {
forAll { (envelope: SuccessEnvelope.Ok[String]) =>
val json = envelope.asJson
val decoded = json.as[SuccessEnvelope.Ok[String]]
decoded shouldBe Right(envelope)
}
}
test("SuccessEnvelope.Created encodes content field") {
forAll { (envelope: SuccessEnvelope.Created[String]) =>
val json = envelope.asJson
json.hcursor.get[String]("content") shouldBe Right(envelope.content)
}
}
test("SuccessEnvelope.Created encodes optional message field") {
forAll { (envelope: SuccessEnvelope.Created[String]) =>
val json = envelope.asJson
envelope.message match {
case Some(msg) => json.hcursor.get[String]("message") shouldBe Right(msg)
case None => json.hcursor.get[Option[String]]("message") shouldBe Right(None)
}
}
}
test("SuccessEnvelope.Created encodes warnings list") {
forAll { (envelope: SuccessEnvelope.Created[String]) =>
val json = envelope.asJson
json.hcursor.get[List[String]]("warnings") shouldBe Right(envelope.warnings)
}
}
test("SuccessEnvelope.Created roundtrips encode/decode") {
forAll { (envelope: SuccessEnvelope.Created[String]) =>
val json = envelope.asJson
val decoded = json.as[SuccessEnvelope.Created[String]]
decoded shouldBe Right(envelope)
}
}
test("SuccessEnvelope.Accepted encodes message field") {
forAll { (envelope: SuccessEnvelope.Accepted) =>
val json = envelope.asJson
json.hcursor.get[String]("message") shouldBe Right(envelope.message)
}
}
test("SuccessEnvelope.Accepted encodes optional monitorUrl field") {
forAll { (envelope: SuccessEnvelope.Accepted) =>
val json = envelope.asJson
envelope.monitorUrl match {
case Some(url) => json.hcursor.get[String]("monitorUrl") shouldBe Right(url)
case None => json.hcursor.get[Option[String]]("monitorUrl") shouldBe Right(None)
}
}
}
test("SuccessEnvelope.Accepted roundtrips encode/decode") {
forAll { (envelope: SuccessEnvelope.Accepted) =>
val json = envelope.asJson
val decoded = json.as[SuccessEnvelope.Accepted]
decoded shouldBe Right(envelope)
}
}
test("SuccessEnvelope.NoContent encodes to unit-like JSON") {
val json = SuccessEnvelope.NoContent.asJson
// NoContent is encoded as unit, which is an empty object
json shouldBe io.circe.Json.obj()
}
test("SuccessEnvelope.NoContent roundtrips encode/decode") {
val json = SuccessEnvelope.NoContent.asJson
val decoded = json.as[SuccessEnvelope.NoContent.type]
decoded shouldBe Right(SuccessEnvelope.NoContent)
}
test("SuccessEnvelope.Ok works with Int content") {
forAll { (envelope: SuccessEnvelope.Ok[Int]) =>
val json = envelope.asJson
json.hcursor.get[Int]("content") shouldBe Right(envelope.content)
json.as[SuccessEnvelope.Ok[Int]] shouldBe Right(envelope)
}
}
test("SuccessEnvelope.Ok works with List[String] content") {
forAll { (envelope: SuccessEnvelope.Ok[List[String]]) =>
val json = envelope.asJson
json.as[SuccessEnvelope.Ok[List[String]]] shouldBe Right(envelope)
}
}
test("SuccessEnvelope.Ok encoder outputs all fields including defaults") {
val envelope = SuccessEnvelope.Ok("test", None, Nil)
val json = envelope.asJson
json.hcursor.get[String]("content") shouldBe Right("test")
json.hcursor.get[Option[String]]("message") shouldBe Right(None)
json.hcursor.get[List[String]]("warnings") shouldBe Right(Nil)
}
test("SuccessEnvelope.Created encoder outputs all fields including defaults") {
val envelope = SuccessEnvelope.Created("test", None, Nil)
val json = envelope.asJson
json.hcursor.get[String]("content") shouldBe Right("test")
json.hcursor.get[Option[String]]("message") shouldBe Right(None)
json.hcursor.get[List[String]]("warnings") shouldBe Right(Nil)
}
test("SuccessEnvelope.Accepted decodes from minimal JSON with defaults applied") {
val minimalJson = io.circe.Json.obj()
val decoded = minimalJson.as[SuccessEnvelope.Accepted]
decoded shouldBe Right(SuccessEnvelope.Accepted())
}
}
================================================
FILE: api/src/test/scala/com/thatdot/api/v2/SuccessEnvelopeGenerators.scala
================================================
package com.thatdot.api.v2
import org.scalacheck.{Arbitrary, Gen}
import com.thatdot.quine.ScalaPrimitiveGenerators
object SuccessEnvelopeGenerators {
import ScalaPrimitiveGenerators.Gens._
object Gens {
val warnings: Gen[List[String]] = smallNonNegNum.flatMap(Gen.listOfN(_, nonEmptyAlphaStr))
def ok[A](contentGen: Gen[A]): Gen[SuccessEnvelope.Ok[A]] =
for {
content <- contentGen
message <- optNonEmptyAlphaStr
warns <- warnings
} yield SuccessEnvelope.Ok(content, message, warns)
def created[A](contentGen: Gen[A]): Gen[SuccessEnvelope.Created[A]] =
for {
content <- contentGen
message <- optNonEmptyAlphaStr
warns <- warnings
} yield SuccessEnvelope.Created(content, message, warns)
val accepted: Gen[SuccessEnvelope.Accepted] =
for {
message <- nonEmptyAlphaStr
monitorUrl <- optNonEmptyAlphaStr
} yield SuccessEnvelope.Accepted(message, monitorUrl)
}
object Arbs {
implicit def okArb[A](implicit arbA: Arbitrary[A]): Arbitrary[SuccessEnvelope.Ok[A]] =
Arbitrary(Gens.ok(arbA.arbitrary))
implicit def createdArb[A](implicit arbA: Arbitrary[A]): Arbitrary[SuccessEnvelope.Created[A]] =
Arbitrary(Gens.created(arbA.arbitrary))
implicit val acceptedArb: Arbitrary[SuccessEnvelope.Accepted] = Arbitrary(Gens.accepted)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/quine/JsonGenerators.scala
================================================
package com.thatdot.quine
import io.circe.Json
import org.scalacheck.{Arbitrary, Gen}
object JsonGenerators {
import ScalaPrimitiveGenerators.Gens.{nonEmptyAlphaStr, smallNonNegNum, smallPosNum}
object Gens {
val nonNullPrimitive: Gen[Json] = Gen.oneOf(
Arbitrary.arbBool.arbitrary.map(Json.fromBoolean),
Arbitrary.arbLong.arbitrary.map(Json.fromLong),
Arbitrary.arbDouble.arbitrary.map(Json.fromDoubleOrNull),
Arbitrary.arbString.arbitrary.map(Json.fromString),
)
val primitive: Gen[Json] = Gen.oneOf(Gen.const(Json.Null), nonNullPrimitive)
def dictionaryOfSize(size: Int): Gen[Map[String, Json]] =
Gen.mapOfN(size, Gen.zip(nonEmptyAlphaStr, primitive))
val dictionary: Gen[Map[String, Json]] = smallNonNegNum.flatMap(dictionaryOfSize)
val nonEmptyDictionary: Gen[Map[String, Json]] = smallPosNum.flatMap(dictionaryOfSize)
val sizedDictionary: Gen[Map[String, Json]] = Gen.sized(dictionaryOfSize)
}
object Arbs {
implicit val primitive: Arbitrary[Json] = Arbitrary(Gens.primitive)
implicit val dictionary: Arbitrary[Map[String, Json]] = Arbitrary(Gens.dictionary)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/quine/ScalaPrimitiveGenerators.scala
================================================
package com.thatdot.quine
import org.scalacheck.{Arbitrary, Gen}
/** Popular primitive-based generators (no `Arbs`; would conflict with ScalaCheck's). */
object ScalaPrimitiveGenerators {
object Gens {
val bool: Gen[Boolean] = Arbitrary.arbitrary[Boolean]
val smallNonNegNum: Gen[Int] = Gen.chooseNum(0, 10)
val smallPosNum: Gen[Int] = Gen.chooseNum(1, 10)
val mediumNonNegNum: Gen[Int] = Gen.chooseNum(0, 1000)
val mediumPosNum: Gen[Int] = Gen.chooseNum(1, 1000)
val largePosNum: Gen[Int] = Gen.chooseNum(1, 1000000)
val port: Gen[Int] = Gen.choose(1, 65535)
val mediumPosLong: Gen[Long] = Gen.chooseNum(1L, 10000L)
val largeNonNegLong: Gen[Long] = Gen.chooseNum(0L, 1000000L)
val largePosLong: Gen[Long] = Gen.chooseNum(1L, 1000000L)
val unitInterval: Gen[Double] = Gen.chooseNum(0.0, 1.0)
val percentage: Gen[Double] = Gen.choose(0.0, 100.0)
val mediumNonNegDouble: Gen[Double] = Gen.chooseNum(0.0, 1000.0)
/** Generates positive integers within the range representable by `2^pow` bits (`1` to `2^pow - 1`).
*
* @param pow the "power" (exponent) of base-2 from which a bit range may be derived (e.g. `7` yields `2^7` or `128` bits)
* @return an integer between `1` and `2^pow - 1`
*/
def numWithinBits(pow: Int): Gen[Int] = Gen.chooseNum(1, (1 << pow) - 1)
val nonEmptyAlphaStr: Gen[String] = Gen.nonEmptyListOf(Gen.alphaChar).map(_.mkString)
val nonEmptyAlphaNumStr: Gen[String] = Gen.nonEmptyListOf(Gen.alphaNumChar).map(_.mkString)
val optNonEmptyAlphaStr: Gen[Option[String]] = Gen.option(nonEmptyAlphaStr)
val optNonEmptyAlphaNumStr: Gen[Option[String]] = Gen.option(nonEmptyAlphaNumStr)
}
}
================================================
FILE: api/src/test/scala/com/thatdot/quine/TimeGenerators.scala
================================================
package com.thatdot.quine
import java.time.Instant
import org.scalacheck.{Arbitrary, Gen}
object TimeGenerators {
object Gens {
/** Generates timestamps from the full possible range. */
val instant: Gen[Instant] = Arbitrary.arbLong.arbitrary.map(Instant.ofEpochMilli)
/** Generates timestamps within a specified range.
*
* @param from
* Optional start of range. Uses `Instant.now()` if not provided and `to` is provided.
* @param to
* Optional end of range. Uses `Instant.now()` if not provided and `from` is provided.
* @return
* A generator for Instants within the range. If neither bound is provided, returns the full-range [[instant]]
* generator.
*/
def instantWithinRange(from: Option[Instant] = None, to: Option[Instant] = None): Gen[Instant] =
(from, to) match {
case (Some(f), Some(t)) => Gen.chooseNum(f.toEpochMilli, t.toEpochMilli).map(Instant.ofEpochMilli)
case (Some(f), None) => Gen.chooseNum(f.toEpochMilli, Instant.now().toEpochMilli).map(Instant.ofEpochMilli)
case (None, Some(t)) => Gen.chooseNum(Instant.now().toEpochMilli, t.toEpochMilli).map(Instant.ofEpochMilli)
case (None, None) => instant
}
}
object Arbs {
implicit val arbInstant: Arbitrary[Instant] = Arbitrary(Gens.instant)
}
}
================================================
FILE: aws/src/main/scala/com/thatdot/aws/model/AwsCredentials.scala
================================================
package com.thatdot.aws.model
import com.thatdot.common.security.Secret
final case class AwsCredentials(accessKeyId: Secret, secretAccessKey: Secret)
================================================
FILE: aws/src/main/scala/com/thatdot/aws/model/AwsRegion.scala
================================================
package com.thatdot.aws.model
final case class AwsRegion(region: String)
================================================
FILE: aws/src/main/scala/com/thatdot/aws/util/AwsOps.scala
================================================
package com.thatdot.aws.util
import scala.reflect.{ClassTag, classTag}
import software.amazon.awssdk.auth.credentials.{
AwsBasicCredentials,
AwsCredentialsProvider,
DefaultCredentialsProvider,
StaticCredentialsProvider,
}
import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder
import software.amazon.awssdk.regions.Region
import com.thatdot.aws.model._
import com.thatdot.common.logging.Log._
import com.thatdot.common.security.Secret
case object AwsOps extends LazySafeLogging {
// the maximum number of simultaneous API requests any individual AWS client should make
// invariant: all AWS clients using HTTP will set this as a maximum concurrency value
val httpConcurrencyPerClient = 100
def staticCredentialsProviderV2(credsOpt: Option[AwsCredentials]): AwsCredentialsProvider =
credsOpt.fold[AwsCredentialsProvider](DefaultCredentialsProvider.builder.build) { credentials =>
import Secret.Unsafe._
StaticCredentialsProvider.create(
AwsBasicCredentials.create(credentials.accessKeyId.unsafeValue, credentials.secretAccessKey.unsafeValue),
)
}
implicit class AwsBuilderOps[Client: ClassTag, Builder <: AwsClientBuilder[Builder, Client]](
builder: AwsClientBuilder[Builder, Client],
) {
/** Credentials to use for this AWS client. If provided, these will be used explicitly.
* If absent, credentials will be inferred from the environment according to AWS's DefaultCredentialsProvider
* This may have security implications! Ensure your environment only contains environment variables,
* java system properties, aws credentials files, and instance profile credentials you trust!
*
* @see https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default
*
* If you are deploying on EC2 and do NOT wish to use EC2 container metadata/credentials, ensure the java property
* `aws.disableEc2Metadata` is set to true, or the environment variable AWS_EC2_METADATA_DISABLED is set to true.
* Note that this will also disable region lookup, and thus require all AWS client constructions to explicitly set
* credentials.
*
* @param credsOpt if set, aws credentials to use explicitly
* @return
*/
def credentialsV2(credsOpt: Option[AwsCredentials]): Builder = {
val creds = credsOpt.orElse {
logger.info(
safe"""No AWS credentials provided while building AWS client of type
|${Safe(classTag[Client].runtimeClass.getSimpleName)}. Defaulting
|to environmental credentials.""".cleanLines,
)
None
}
builder.credentialsProvider(staticCredentialsProviderV2(creds))
}
def regionV2(regionOpt: Option[AwsRegion]): Builder =
regionOpt.fold {
logger.info(
safe"""No AWS region provided while building AWS client of type:
|${Safe(classTag[Client].runtimeClass.getSimpleName)}.
|Defaulting to environmental settings.""".cleanLines,
)
builder.applyMutation(_ => ()) // return the builder unmodified
}(region => builder.region(Region.of(region.region)))
}
}
================================================
FILE: aws/src/test/scala/com/thatdot/aws/util/AwsOpsSpec.scala
================================================
package com.thatdot.aws.util
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
import com.thatdot.aws.model.AwsCredentials
import com.thatdot.common.security.Secret
class AwsOpsSpec extends AnyWordSpec with Matchers {
"staticCredentialsProviderV2" should {
"extract actual Secret values for SDK usage" in {
val accessKeyId = "AKIAIOSFODNN7EXAMPLE"
val secretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
val credentials = AwsCredentials(
accessKeyId = Secret(accessKeyId),
secretAccessKey = Secret(secretAccessKey),
)
val provider = AwsOps.staticCredentialsProviderV2(Some(credentials))
val resolved = provider.resolveCredentials()
resolved.accessKeyId() shouldBe accessKeyId
resolved.secretAccessKey() shouldBe secretAccessKey
}
"return DefaultCredentialsProvider when credentials are None" in {
val provider = AwsOps.staticCredentialsProviderV2(None)
provider shouldBe a[DefaultCredentialsProvider]
}
"preserve credential values through Secret wrapper" in {
val testCases = Seq(
("AKIA123", "secret123"),
("AKIASPECIAL!@#$%", "secret/with+special=chars"),
("A" * 20, "B" * 40),
)
for ((accessKey, secretKey) <- testCases) {
val credentials = AwsCredentials(Secret(accessKey), Secret(secretKey))
val provider = AwsOps.staticCredentialsProviderV2(Some(credentials))
val resolved = provider.resolveCredentials()
withClue(s"For accessKey=$accessKey, secretKey=$secretKey: ") {
resolved.accessKeyId() shouldBe accessKey
resolved.secretAccessKey() shouldBe secretKey
}
}
}
}
}
================================================
FILE: build.sbt
================================================
import Dependencies.*
import scalajsbundler.util.JSON._
import QuineSettings.*
ThisBuild / resolvers += "thatDot maven" at "https://s3.us-west-2.amazonaws.com/com.thatdot.dependencies/release/"
ThisBuild / scalaVersion := scalaV
addCommandAlias("fmtall", "; scalafmtAll; scalafmtSbt")
addCommandAlias("fixall", "; scalafixAll; fmtall")
ThisBuild / evictionErrorLevel := Level.Info
Global / concurrentRestrictions := Seq(
Tags.limit(Tags.Test, 1),
)
// Core streaming graph interpreter
lazy val `quine-core`: Project = project
.settings(commonSettings)
.dependsOn(`quine-language`)
.settings(
libraryDependencies ++= Seq(
"org.graalvm.js" % "js" % graalV,
"com.chuusai" %% "shapeless" % shapelessV,
"org.apache.pekko" %% "pekko-actor" % pekkoV,
"org.apache.pekko" %% "pekko-stream" % pekkoV,
"org.apache.pekko" %% "pekko-slf4j" % pekkoV,
"com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingV,
"io.dropwizard.metrics" % "metrics-core" % dropwizardMetricsV,
"io.circe" %% "circe-parser" % circeV,
"org.msgpack" % "msgpack-core" % msgPackV,
"org.apache.commons" % "commons-text" % commonsTextV,
"com.github.blemale" %% "scaffeine" % scaffeineV,
"io.github.hakky54" % "ayza" % ayzaV,
"org.typelevel" %% "cats-core" % catsV,
"org.typelevel" %% "cats-effect" % catsEffectV,
"com.thatdot" %% "quine-id" % quineCommonV,
"com.lihaoyi" %% "pprint" % pprintV,
"commons-codec" % "commons-codec" % commonsCodecV,
// Testing
"org.scalatest" %% "scalatest" % scalaTestV % Test,
"org.scalacheck" %% "scalacheck" % scalaCheckV % Test,
"org.scalatestplus" %% "scalacheck-1-17" % scalaTestScalaCheckV % Test,
"org.apache.pekko" %% "pekko-testkit" % pekkoTestkitV % Test,
"ch.qos.logback" % "logback-classic" % logbackV % Test,
"commons-io" % "commons-io" % commonsIoV % Test,
),
// Compile different files depending on scala version
Compile / unmanagedSourceDirectories += {
(Compile / sourceDirectory).value / "scala-2.13"
},
addCompilerPlugin("org.typelevel" %% "kind-projector" % kindProjectorV cross CrossVersion.full),
// Uncomment the following 2 lines to generate flamegraphs for the project's compilation in target/scala-2.13/classes/META-INF
// (look for `.flamegraph` files -- these may be imported into intellij profiler or flamegraph.pl)
// ThisBuild / scalacOptions += "-Vstatistics",
// addCompilerPlugin("ch.epfl.scala" %% "scalac-profiling" % "1.1.0-RC3" cross CrossVersion.full)
)
.enablePlugins(BuildInfoPlugin, FlatcPlugin)
.settings(
// Allow BuildInfo to be cached on `-DIRTY` versions, to avoid recompilation during development
buildInfoOptions := (if (git.gitUncommittedChanges.value) Seq() else Seq(BuildInfoOption.BuildTime)),
buildInfoKeys := Seq[BuildInfoKey](
version,
git.gitHeadCommit,
git.gitUncommittedChanges,
git.gitHeadCommitDate,
BuildInfoKey.action("javaVmName")(scala.util.Properties.javaVmName),
BuildInfoKey.action("javaVendor")(scala.util.Properties.javaVendor),
BuildInfoKey.action("javaVersion")(scala.util.Properties.javaVersion),
),
buildInfoPackage := "com.thatdot.quine",
)
// Quine Language - Cypher parser and language services
lazy val `quine-language`: Project = project
.settings(commonSettings)
.enablePlugins(Antlr4Plugin)
.settings(
libraryDependencies ++= Seq(
"org.antlr" % "antlr4-runtime" % antlr4RuntimeV,
"org.typelevel" %% "cats-effect" % catsEffectV,
"org.eclipse.lsp4j" % "org.eclipse.lsp4j" % lsp4jV,
"com.chuusai" %% "shapeless" % shapelessV,
"com.google.guava" % "guava" % guavaV,
"com.47deg" %% "memeid4s" % memeid4sV,
"com.thatdot" %% "quine-id" % quineCommonV,
"com.thatdot" %% "quine-utils" % quineCommonV,
// Testing
"org.scalameta" %% "munit" % munitV % Test,
),
Antlr4 / antlr4PackageName := Some("com.thatdot.quine.cypher.parsing"),
Antlr4 / antlr4Version := antlr4RuntimeV,
Antlr4 / antlr4GenListener := false,
Antlr4 / antlr4GenVisitor := true,
testFrameworks += new TestFramework("munit.Framework"),
addCompilerPlugin("org.typelevel" %% "kind-projector" % kindProjectorV cross CrossVersion.full),
)
lazy val `quine-serialization`: Project = project
.settings(commonSettings)
.dependsOn(
`data`,
`quine-core` % "compile->compile;test->test",
)
.settings(
libraryDependencies ++= Seq(
"com.google.api.grpc" % "proto-google-common-protos" % protobufCommonV,
"com.google.protobuf" % "protobuf-java" % protobufV,
"software.amazon.glue" % "schema-registry-serde" % amazonGlueV, // for its protobuf DynamicSchema utility
// Glue->AWS Netty Client->Netty, which has some CVEs. Glue 1.1.27 has vulnerable Netty; override to safe AWS SDK.
"software.amazon.awssdk" % "netty-nio-client" % awsSdkV,
"org.apache.avro" % "avro" % avroV,
"org.endpoints4s" %%% "json-schema-generic" % endpoints4sDefaultV,
"org.endpoints4s" %%% "json-schema-circe" % endpoints4sCirceV,
),
)
// MapDB implementation of a Quine persistor
lazy val `quine-mapdb-persistor`: Project = project
.settings(commonSettings)
.dependsOn(`quine-core` % "compile->compile;test->test")
.settings(
/* `net.jpountz.lz4:lz4` was moved to `org.lz4:lz4-java`, then to
* `at.yawk.lz4:lz4-java` (the maintained fork). MapDB still depends on the
* old coordinates, so we exclude the old JAR and pull in the current one.
*/
libraryDependencies ++= Seq(
("org.mapdb" % "mapdb" % mapDbV).exclude("net.jpountz.lz4", "lz4"),
"at.yawk.lz4" % "lz4-java" % lz4JavaV,
),
)
// RocksDB implementation of a Quine persistor
lazy val `quine-rocksdb-persistor`: Project = project
.settings(commonSettings)
.dependsOn(`quine-core` % "compile->compile;test->test")
.settings(
libraryDependencies ++= Seq(
"org.rocksdb" % "rocksdbjni" % rocksdbV,
),
)
// Cassandra implementation of a Quine persistor
lazy val `quine-cassandra-persistor`: Project = project
.configs(Integration)
.settings(commonSettings, integrationSettings)
.dependsOn(`quine-core` % "compile->compile;test->test")
.enablePlugins(spray.boilerplate.BoilerplatePlugin)
.settings(
libraryDependencies ++= Seq(
"org.typelevel" %% "cats-core" % catsV,
"org.apache.cassandra" % "java-driver-query-builder" % cassandraClientV,
// The org name for the Cassandra java-driver was changed from com.datastax.oss to org.apache.cassandra
// The sigv4-auth plugin specifies a dep on com.datastax.oss, SBT doesn't know that our org.apache.cassandra
// dep is supposed to be the replacement for that, and includes both on the classpath, which then conflict
// at the sbt-assembly step (because they both have the same package names internally).
"software.aws.mcs" % "aws-sigv4-auth-cassandra-java-driver-plugin" % sigv4AuthCassandraPluginV exclude ("com.datastax.oss", "java-driver-core"),
"software.amazon.awssdk" % "sts" % awsSdkV,
"com.github.nosan" % "embedded-cassandra" % embeddedCassandraV % Test,
),
)
// Parser and interpreter for a subset of [Gremlin](https://tinkerpop.apache.org/gremlin.html)
lazy val `quine-gremlin`: Project = project
.settings(commonSettings)
.dependsOn(`quine-core` % "compile->compile;test->test")
.settings(
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-parser-combinators" % scalaParserCombinatorsV,
"org.apache.commons" % "commons-text" % commonsTextV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
),
)
// Compiler for compiling [Cypher](https://neo4j.com/docs/cypher-manual/current/) into Quine queries
lazy val `quine-cypher`: Project = project
.settings(commonSettings)
.dependsOn(`quine-core` % "compile->compile;test->test")
.settings(
scalacOptions ++= Seq(
"-language:reflectiveCalls",
"-Xlog-implicits",
),
libraryDependencies ++= Seq(
"com.thatdot.opencypher" %% "expressions" % openCypherV,
"com.thatdot.opencypher" %% "front-end" % openCypherV,
"com.thatdot.opencypher" %% "opencypher-cypher-ast-factory" % openCypherV,
"com.thatdot.opencypher" %% "util" % openCypherV,
"org.typelevel" %% "cats-core" % catsV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
"org.apache.pekko" %% "pekko-stream-testkit" % pekkoV % Test,
),
addCompilerPlugin("org.typelevel" % "kind-projector" % kindProjectorV cross CrossVersion.full),
addCompilerPlugin("com.olegpy" %% "better-monadic-for" % betterMonadicForV),
)
/*
* Version 7.5.1. It is expected that `Network` and `DataSet` are available under
* A globally available `vis` object, as with
*
* ```html
* <script
* type="text/javascript"
* src="https://unpkg.com/vis-network/standalone/umd/vis-network.min.js"
* ></script>
* ```
*
* Thanks to [`scala-js-ts-importer`][ts-importer] which made it possible to generate
* A first pass of the facade directly from the Typescipt bindings provided with
* `vis-network` (see `Network.d.ts`).
*
* [ts-importer]: https://github.com/sjrd/scala-js-ts-importer
* [visjs]: https://github.com/visjs/vis-network
*/
lazy val `visnetwork-facade`: Project = project
.settings(commonSettings)
.enablePlugins(ScalaJSPlugin)
.settings(
libraryDependencies ++= Seq(
"org.scala-js" %%% "scalajs-dom" % scalajsDomV,
),
)
lazy val `aws`: Project = project
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.thatdot" %% "quine-logging" % quineCommonV,
"com.thatdot" %% "quine-security" % quineCommonV,
"software.amazon.awssdk" % "aws-core" % awsSdkV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
),
)
lazy val `data`: Project = project
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.thatdot" %% "quine-logging" % quineCommonV,
"com.thatdot" %% "quine-utils" % quineCommonV,
"com.google.protobuf" % "protobuf-java" % protobufV,
"io.circe" %% "circe-core" % circeV,
"org.apache.avro" % "avro" % avroV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
),
)
/** V2 API type definitions shared between server (JVM) and browser (ScalaJS). */
lazy val `quine-endpoints2` = crossProject(JSPlatform, JVMPlatform)
.crossType(CrossType.Pure)
.in(file("quine-endpoints2"))
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.softwaremill.sttp.tapir" %%% "tapir-core" % tapirV,
"io.circe" %%% "circe-core" % circeV,
"io.circe" %%% "circe-generic-extras" % circeGenericExtrasV,
),
)
lazy val `api`: Project = project
.in(file("api"))
.settings(commonSettings)
.dependsOn(`quine-serialization`, `quine-endpoints2`.jvm)
.settings(
libraryDependencies ++= Seq(
"com.thatdot" %% "quine-security" % quineCommonV,
"com.softwaremill.sttp.tapir" %% "tapir-core" % tapirV,
"com.softwaremill.sttp.tapir" %% "tapir-json-circe" % tapirV,
"io.circe" %% "circe-core" % circeV,
"io.circe" %% "circe-generic-extras" % circeGenericExtrasV,
"io.circe" %% "circe-yaml" % circeYamlV,
"com.thatdot" %% "quine-security" % quineCommonV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
"org.scalatestplus" %% "scalacheck-1-17" % scalaTestScalaCheckV % Test,
),
)
lazy val `outputs2`: Project = project
.settings(commonSettings)
.dependsOn(`aws`, `data`, `quine-core`, `quine-serialization`)
.settings(
libraryDependencies ++= Seq(
"com.thatdot" %% "quine-logging" % quineCommonV,
"org.apache.pekko" %% "pekko-actor" % pekkoV,
"org.apache.pekko" %% "pekko-stream" % pekkoV,
"org.apache.pekko" %% "pekko-http" % pekkoHttpV,
"org.apache.pekko" %% "pekko-connectors-kafka" % pekkoKafkaV,
"org.apache.pekko" %% "pekko-connectors-kinesis" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-sns" % pekkoConnectorsV,
"software.amazon.awssdk" % "netty-nio-client" % awsSdkV,
"com.google.protobuf" % "protobuf-java" % protobufV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
"org.scalacheck" %%% "scalacheck" % scalaCheckV % Test,
"org.apache.pekko" %% "pekko-http-testkit" % pekkoHttpV % Test,
),
)
/** V1 API definitions (that may be used for internal modeling at times) for `quine`-based applications */
lazy val `quine-endpoints` = crossProject(JSPlatform, JVMPlatform)
.crossType(CrossType.Pure)
.in(file("quine-endpoints"))
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.thatdot" %%% "quine-security" % quineCommonV,
"org.endpoints4s" %%% "json-schema-generic" % endpoints4sDefaultV,
"org.endpoints4s" %%% "json-schema-circe" % endpoints4sCirceV,
"io.circe" %% "circe-core" % circeV,
"org.endpoints4s" %%% "openapi" % endpoints4sOpenapiV,
"com.lihaoyi" %% "ujson-circe" % ujsonCirceV, // For the OpenAPI rendering
"org.scalacheck" %%% "scalacheck" % scalaCheckV % Test,
"org.scalatest" %%% "scalatest" % scalaTestV % Test,
"com.softwaremill.sttp.tapir" %% "tapir-core" % tapirV, // For tapir annotations
),
)
.jsSettings(
// Provides an implementation that allows us to use java.time.Instant in Scala.js
libraryDependencies += "io.github.cquiroz" %%% "scala-java-time" % scalaJavaTimeV,
)
/** Contains the common (among product needs) converters/conversions between
* the independent definitions of API models and internal models. Notably
* not versioned because versioning of API and internal models are independent.
*/
lazy val `model-converters`: Project = project
.settings(commonSettings)
.dependsOn(
`api`,
`outputs2`,
`quine-endpoints`.jvm,
)
// Quine web application
lazy val `quine-browser`: Project = project
.settings(commonSettings, visNetworkSettings)
.dependsOn(`quine-endpoints`.js, `visnetwork-facade`, `quine-endpoints2`.js)
.enablePlugins(ScalaJSBundlerPlugin)
.settings(
libraryDependencies ++= Seq(
"org.scala-js" %%% "scalajs-dom" % scalajsDomV,
"org.scala-js" %%% "scala-js-macrotask-executor" % scalajsMacroTaskExecutorV,
"org.endpoints4s" %%% "xhr-client" % endpoints4sXhrClientV,
"io.circe" %%% "circe-generic" % circeV,
"io.circe" %%% "circe-parser" % circeV,
"com.raquo" %%% "laminar" % laminarV,
"com.raquo" %%% "waypoint" % waypointV,
),
Compile / npmDevDependencies ++= Seq(
// When updating, check whether the minimatch yarn resolution below is still needed
"ts-loader" -> "8.0.0",
"typescript" -> "4.9.5",
"@types/node" -> "16.7.13",
// Webpack 5 loaders and polyfills (required by common.webpack.config.js)
"style-loader" -> "3.3.4",
"css-loader" -> "6.11.0",
"buffer" -> "6.0.3",
"stream-browserify" -> "3.0.0",
"path-browserify" -> "1.0.1",
"process" -> "0.11.10",
),
Compile / npmDependencies ++= Seq(
"es6-shim" -> "0.35.7",
"plotly.js" -> s"npm:plotly.js-strict-dist-min@${plotlyV}", // CSP-compliant strict bundle
"@stoplight/elements" -> stoplightElementsV,
"react" -> reactV, // Peer dependency of @stoplight/elements
"react-dom" -> reactV,
"mkdirp" -> "1.0.0",
"@coreui/coreui" -> coreuiV,
"@coreui/icons" -> coreuiIconsV,
"@popperjs/core" -> "2.11.8",
),
// Force patched dependency versions via yarn resolutions (see NPM Override Versions in Dependencies.scala)
Compile / additionalNpmConfig := Map(
"resolutions" -> obj(
"lodash" -> str(lodashV),
"react-router" -> str(reactRouterV),
"react-router-dom" -> str(reactRouterV),
"@remix-run/router" -> str(remixRunRouterV),
"minimatch" -> str(minimatchV),
"yaml" -> str(yamlV),
"brace-expansion" -> str(braceExpansionV),
),
),
webpackNodeArgs := nodeLegacySslIfAvailable,
// Scalajs-bundler 0.21.1 updates to webpack 5 but doesn't inform webpack that the scalajs-based file it emits is
// an entrypoint -- therefore webpack emits an error saying effectively, "no entrypoint" that we must ignore.
// This aggressively ignores all warnings from webpack, which is more than necessary, but trivially works
webpackExtraArgs := Seq("--ignore-warnings-message", "/.*/"),
fastOptJS / webpackConfigFile := Some(baseDirectory.value / "dev.webpack.config.js"),
fastOptJS / webpackDevServerExtraArgs := Seq("--inline", "--hot"),
fullOptJS / webpackConfigFile := Some(baseDirectory.value / "prod.webpack.config.js"),
Test / webpackConfigFile := Some(baseDirectory.value / "common.webpack.config.js"),
test := {},
useYarn := true,
yarnExtraArgs := Seq("--frozen-lockfile"),
)
// Streaming graph application built on top of the Quine library
lazy val `quine`: Project = project
.settings(commonSettings)
.dependsOn(
`quine-core` % "compile->compile;test->test",
`quine-cypher` % "compile->compile;test->test",
`quine-endpoints`.jvm % "compile->compile;test->test",
`data` % "compile->compile;test->test",
`api` % "compile->compile;test->test",
`model-converters`,
`outputs2` % "compile->compile;test->test",
`quine-gremlin`,
`quine-cassandra-persistor`,
`quine-mapdb-persistor`,
`quine-rocksdb-persistor`,
)
.settings(
libraryDependencies ++= Seq(
"ch.qos.logback" % "logback-classic" % logbackV,
"com.github.davidb" % "metrics-influxdb" % metricsInfluxdbV,
"com.github.jnr" % "jnr-posix" % jnrPosixV,
"com.github.pjfanning" %% "pekko-http-circe" % pekkoHttpCirceV,
"com.github.pureconfig" %% "pureconfig" % pureconfigV,
"com.github.scopt" %% "scopt" % scoptV,
"com.google.api.grpc" % "proto-google-common-protos" % protobufCommonV,
"com.github.ben-manes.caffeine" % "caffeine" % caffeineV,
"com.github.blemale" %% "scaffeine" % scaffeineV,
"com.google.protobuf" % "protobuf-java" % protobufV,
"com.softwaremill.sttp.tapir" %% "tapir-pekko-http-server" % tapirV,
"com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % tapirV,
"com.softwaremill.sttp.tapir" %% "tapir-json-circe" % tapirV,
"com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % openApiCirceYamlV exclude ("io.circe", "circe-yaml"),
"org.apache.pekko" %% "pekko-http-testkit" % pekkoHttpV % Test,
"io.circe" %% "circe-yaml" % circeYamlV,
"com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingV,
"ch.qos.logback" % "logback-classic" % logbackV,
"com.softwaremill.sttp.tapir" %% "tapir-sttp-stub-server" % tapirV % Test,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
//"commons-io" % "commons-io" % commonsIoV % Test,
"io.circe" %% "circe-config" % "0.10.2",
"io.circe" %% "circe-generic-extras" % circeGenericExtrasV,
"io.circe" %% "circe-yaml-v12" % "0.16.1",
"io.circe" %% "circe-core" % circeV,
"io.dropwizard.metrics" % "metrics-core" % dropwizardMetricsV,
"io.dropwizard.metrics" % "metrics-jmx" % dropwizardMetricsV,
"io.dropwizard.metrics" % "metrics-jvm" % dropwizardMetricsV,
"org.apache.commons" % "commons-csv" % apacheCommonsCsvV,
"org.apache.kafka" % "kafka-clients" % kafkaClientsV,
"org.apache.pekko" %% "pekko-connectors-csv" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-kafka" % pekkoKafkaV,
"org.apache.pekko" %% "pekko-connectors-kinesis" % pekkoConnectorsV exclude ("org.rocksdb", "rocksdbjni"),
"software.amazon.kinesis" % "amazon-kinesis-client" % amazonKinesisClientV,
"org.apache.pekko" %% "pekko-connectors-s3" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-sns" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-sqs" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-sse" % pekkoConnectorsV,
"org.apache.pekko" %% "pekko-connectors-text" % pekkoConnectorsV,
// pekko-http-xml is not a direct dep, but an older version is pulled in transitively by
// pekko-connectors-s3 above. All pekko-http module version numbers need to match exactly, or else it throws
// at startup: "java.lang.IllegalStateException: Detected possible incompatible versions on the classpath."
"org.apache.pekko" %% "pekko-http-xml" % pekkoHttpV,
"org.apache.pekko" %% "pekko-stream-testkit" % pekkoV % Test,
"org.endpoints4s" %% "pekko-http-server" % endpoints4sHttpServerV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
"org.scalatestplus" %% "scalacheck-1-17" % scalaTestScalaCheckV % Test,
// WebJars (javascript dependencies masquerading as JARs)
"org.webjars" % "ionicons" % ioniconsV,
"org.webjars" % "jquery" % jqueryV,
"org.webjars" % "webjars-locator" % webjarsLocatorV,
"org.webjars.npm" % "sugar-date" % sugarV,
"org.apache.avro" % "avro" % avroV,
// AWS SDK deps (next 4) effectively bundle sibling JARs needed for certain features, despite no code references
"software.amazon.awssdk" % "sso" % awsSdkV,
"software.amazon.awssdk" % "ssooidc" % awsSdkV,
"software.amazon.awssdk" % "sts" % awsSdkV,
"software.amazon.awssdk" % "aws-query-protocol" % awsSdkV,
),
// Add JVM options for tests to allow reflection access to java.util (needed for env var manipulation in tests)
Test / javaOptions += "--add-opens=java.base/java.util=ALL-UNNAMED",
Test / fork := true,
)
.enablePlugins(WebScalaJSBundlerPlugin)
.settings(
scalaJSProjects := Seq(`quine-browser`),
Assets / pipelineStages := Seq(scalaJSPipeline),
)
.enablePlugins(BuildInfoPlugin, Packaging, Docker, Ecr)
.settings(
startupMessage := "",
buildInfoKeys := Seq[BuildInfoKey](version, startupMessage),
buildInfoPackage := "com.thatdot.quine.app",
)
lazy val `quine-docs`: Project = {
val docJsonV1 = Def.setting((Compile / sourceManaged).value / "reference" / "openapi-v1.json")
val docJsonV2 = Def.setting((Compile / sourceManaged).value / "reference" / "openapi-v2.json")
val cypherTable1 = Def.setting((Compile / sourceManaged).value / "reference" / "cypher-builtin-functions.md")
val cypherTable2 =
Def.setting((Compile / sourceManaged).value / "reference" / "cypher-user-defined-functions.md")
val cypherTable3 =
Def.setting((Compile / sourceManaged).value / "reference" / "cypher-user-defined-procedures.md")
val generateDocs = TaskKey[Unit]("generateDocs", "Generate documentation tables for the Quine (Mkdocs) project")
Project("quine-docs", file("quine-docs"))
.dependsOn(`quine`)
.settings(commonSettings)
.settings(
generateDocs := Def
.sequential(
Def.taskDyn {
(Compile / runMain)
.toTask(
List(
" com.thatdot.quine.docs.GenerateCypherTables",
cypherTable1.value.getAbsolutePath,
cypherTable2.value.getAbsolutePath,
cypherTable3.value.getAbsolutePath,
).mkString(" "),
)
},
Def.taskDyn {
(Compile / runMain)
.toTask(s" com.thatdot.quine.docs.GenerateOpenApi ${docJsonV1.value.getAbsolutePath}")
},
Def.taskDyn {
(Compile / runMain)
.toTask(s" com.thatdot.quine.docs.GenerateOpenApiV2 ${docJsonV2.value.getAbsolutePath}")
},
)
.value,
)
.settings(
libraryDependencies ++= Seq(
"org.pegdown" % "pegdown" % pegdownV,
"org.parboiled" % "parboiled-java" % parboiledV,
"org.scalatest" %% "scalatest" % scalaTestV % Test,
),
)
}
// Spurious warnings
Global / excludeLintKeys += `quine-browser` / webpackNodeArgs
Global / excludeLintKeys += `quine-browser` / webpackExtraArgs
================================================
FILE: data/src/main/scala/com/thatdot/data/DataFoldableFrom.scala
================================================
package com.thatdot.data
import scala.collection.{SeqView, View, mutable}
import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
import scala.util.Try
import org.apache.pekko.util
import com.google.protobuf.Descriptors.EnumValueDescriptor
import com.google.protobuf.Descriptors.FieldDescriptor.JavaType
import com.google.protobuf.{ByteString, Descriptors, DynamicMessage}
import io.circe.{Json, JsonNumber, JsonObject}
import org.apache.avro.generic.{GenericArray, GenericEnumSymbol, GenericFixed, GenericRecord}
import com.thatdot.common.logging.Log._
trait DataFoldableFrom[A] extends LazySafeLogging {
def fold[B](value: A, folder: DataFolderTo[B]): B
def fold[B, Frame](t: (() => Try[A], Frame), folder: DataFolderTo[B]): (Try[B], Frame) =
(t._1().map(a => fold(a, folder)), t._2)
def to[B: DataFolderTo: ClassTag]: A => B = {
case b: B => b
case a => fold(a, DataFolderTo[B])
}
}
object DataFoldableFrom {
def apply[A](implicit df: DataFoldableFrom[A]): DataFoldableFrom[A] = df
def contramap[A: DataFoldableFrom, B](f: B => A): DataFoldableFrom[B] =
new DataFoldableFrom[B] {
override def fold[C](value: B, folder: DataFolderTo[C]): C =
DataFoldableFrom[A].fold(f(value), folder)
}
implicit final class Ops[A](private val self: DataFoldableFrom[A]) extends AnyVal {
def contramap[B](f: B => A): DataFoldableFrom[B] =
DataFoldableFrom.contramap(f)(self)
}
implicit val jsonDataFoldable: DataFoldableFrom[Json] = new DataFoldableFrom[Json] {
def fold[B](value: Json, folder: DataFolderTo[B]): B =
value.foldWith(new Json.Folder[B] {
def onNull: B = folder.nullValue
def onBoolean(value: Boolean): B = if (value) folder.trueValue else folder.falseValue
def onNumber(value: JsonNumber): B =
value.toLong.fold(folder.floating(value.toDouble))(l => folder.integer(l))
def onString(value: String): B = folder.string(value)
def onArray(value: Vector[Json]): B = {
val builder = folder.vectorBuilder()
value.foreach(j => builder.add(fold[B](j, folder)))
builder.finish()
}
def onObject(value: JsonObject): B = {
val builder = folder.mapBuilder()
value.toIterable.foreach { case (k, v) => builder.add(k, fold[B](v, folder)) }
builder.finish()
}
})
}
implicit val byteStringDataFoldable: DataFoldableFrom[util.ByteString] = new DataFoldableFrom[util.ByteString] {
def fold[B](value: util.ByteString, folder: DataFolderTo[B]): B =
folder.bytes(value.toArrayUnsafe())
}
implicit val bytesDataFoldable: DataFoldableFrom[Array[Byte]] = new DataFoldableFrom[Array[Byte]] {
def fold[B](value: Array[Byte], folder: DataFolderTo[B]): B =
folder.bytes(value)
}
implicit val stringDataFoldable: DataFoldableFrom[String] = new DataFoldableFrom[String] {
def fold[B](value: String, folder: DataFolderTo[B]): B =
folder.string(value)
}
implicit val stringIterableDataFoldable: DataFoldableFrom[Iterable[String]] = new DataFoldableFrom[Iterable[String]] {
override def fold[B](value: Iterable[String], folder: DataFolderTo[B]): B = {
val builder = folder.vectorBuilder()
value.foreach(v => builder.add(folder.string(v)))
builder.finish()
}
}
implicit val stringVectorDataFoldable: DataFoldableFrom[Vector[String]] = new DataFoldableFrom[Vector[String]] {
override def fold[B](value: Vector[String], folder: DataFolderTo[B]): B = {
val builder = folder.vectorBuilder()
value.foreach(v => builder.add(folder.string(v)))
builder.finish()
}
}
implicit val stringMapDataFoldable: DataFoldableFrom[Map[String, String]] =
new DataFoldableFrom[Map[String, String]] {
override def fold[B](value: Map[String, String], folder: DataFolderTo[B]): B = {
val builder = folder.mapBuilder()
value.foreach { case (name, value) =>
builder.add(name, folder.string(value))
}
builder.finish()
}
}
implicit val protobufDataFoldable: DataFoldableFrom[DynamicMessage] = new DataFoldableFrom[DynamicMessage] {
import com.google.protobuf.Descriptors.FieldDescriptor.JavaType._
private def fieldToValue[B](javaType: JavaType, value: AnyRef, folder: DataFolderTo[B]): B =
javaType match {
case STRING => folder.string(value.asInstanceOf[String])
case INT | LONG => folder.integer(value.asInstanceOf[java.lang.Number].longValue)
case FLOAT | DOUBLE => folder.floating(value.asInstanceOf[java.lang.Number].doubleValue)
case BOOLEAN =>
val bool = value.asInstanceOf[java.lang.Boolean]
if (bool) folder.trueValue else folder.falseValue
case BYTE_STRING => folder.bytes(value.asInstanceOf[ByteString].toByteArray)
case ENUM => folder.string(value.asInstanceOf[EnumValueDescriptor].getName)
case MESSAGE => fold(value.asInstanceOf[DynamicMessage], folder)
}
override def fold[B](message: DynamicMessage, folder: DataFolderTo[B]): B = {
val descriptor: Descriptors.Descriptor = message.getDescriptorForType
val oneOfs: SeqView[Descriptors.OneofDescriptor] = descriptor.getOneofs.asScala.view
// optionals are modeled as (synthetic) oneOfs of a single field.
// Kind of annoying finding a replacement for isSynthetic: https://github.com/googleapis/sdk-platform-java/pull/2764
val (optionals, realOneOfs) = oneOfs.partition { oneof =>
// `getRealContainingOneof` call ends up being `null` if the `oneof` is synthetic,
// with a use of `isSynthetic` in its implementation.
// There might be a case where a user really has a `oneof` with a single optional
// field, so I did not use isOptional here.
oneof.getField(0).getRealContainingOneof == null
}
// synthetic oneOfs (optionals) just have the one field
val setOptionals: View[Descriptors.FieldDescriptor] = optionals.map(_.getField(0)).filter(message.hasField)
// Find which field in each oneOf is set
val oneOfFields: View[Descriptors.FieldDescriptor] =
realOneOfs.flatMap(_.getFields.asScala.find(message.hasField))
val regularFields = descriptor.getFields.asScala.view diff oneOfs.flatMap(_.getFields.asScala).toVector
val mapBuilder: DataFolderTo.MapBuilder[B] = folder.mapBuilder()
(setOptionals ++ oneOfFields ++ regularFields).foreach { field =>
val b: B = {
if (field.isRepeated) {
if (field.isMapField) {
val localMapBuilder = folder.mapBuilder()
message
.getField(field)
.asInstanceOf[java.util.List[DynamicMessage]]
.asScala
.foreach { mapEntry =>
/*
mapEntry.getDescriptorForType is a type described as:
message MapFieldEntry {
key_type key = 1;
value_type value = 2;
}
We already know what fields it contains.
*/
val buffer: mutable.Buffer[Descriptors.FieldDescriptor] =
mapEntry.getDescriptorForType.getFields.asScala
assert(buffer.length == 2)
val k = buffer.head
val v = buffer.tail.head
assert(k.getName == "key")
assert(v.getName == "value")
val maybeKey = k.getJavaType match {
// According to Protobuf docs, "the key_type can be any integral or string type"
// https://developers.google.com/protocol-buffers/docs/proto3#maps
case STRING => Some(mapEntry.getField(k).asInstanceOf[String])
case INT | LONG | BOOLEAN => Some(mapEntry.getField(k).toString)
case other =>
logger.warn(
safe"Cannot process the key ${Safe(other.toString)}. Protobuf can only accept keys of type String, Boolean, Integer. This map key will be ignored.",
)
None
}
maybeKey.map(key =>
localMapBuilder.add(key, fieldToValue(v.getJavaType, mapEntry.getField(v), folder)),
)
}
localMapBuilder.finish()
} else {
val vecBuilder = folder.vectorBuilder()
message
.getField(field)
.asInstanceOf[java.util.List[AnyRef]]
.asScala
.map(f => fieldToValue(field.getJavaType, f, folder))
.foreach(vecBuilder.add)
vecBuilder.finish()
}
} else {
fieldToValue(field.getJavaType, message.getField(field), folder)
}
}
mapBuilder.add(field.getName, b)
}
mapBuilder.finish()
}
}
implicit val avroDataFoldable: DataFoldableFrom[GenericRecord] = new DataFoldableFrom[GenericRecord] {
private def foldMapLike[B](kv: Iterable[(String, Any)], folder: DataFolderTo[B]): B = {
val mapBuilder = folder.mapBuilder()
kv.foreach { case (k, v) => mapBuilder.add(k, foldField(v, folder)) }
mapBuilder.finish()
}
// All of the underlying types for avro were taken from here: https://stackoverflow.com/questions/34070028/get-a-typed-value-from-an-avro-genericrecord/34234039#34234039
private def foldField[B](field: Any, folder: DataFolderTo[B]): B = field match {
case b: java.lang.Boolean if b => folder.trueValue
case b: java.lang.Boolean if !b => folder.falseValue
case i: java.lang.Integer => folder.integer(i.longValue)
case i: java.lang.Long => folder.integer(i)
case f: java.lang.Float => folder.floating(f.doubleValue)
case d: java.lang.Double => folder.floating(d)
case bytes: java.nio.ByteBuffer => folder.bytes(bytes.array)
case str: CharSequence => folder.string(str.toString)
case record: GenericRecord =>
foldMapLike(
record.getSchema.getFields.asScala.collect {
case k if record.hasField(k.name) => (k.name, record.get(k.name))
},
folder,
)
case map: java.util.Map[_, _] => foldMapLike(map.asScala.map { case (k, v) => (k.toString, v) }, folder)
case symbol: GenericEnumSymbol[_] => folder.string(symbol.toString)
case array: GenericArray[_] =>
val vector = folder.vectorBuilder()
array.forEach(elem => vector.add(foldField(elem, folder)))
vector.finish()
case fixed: GenericFixed => folder.bytes(fixed.bytes)
case n if n == null => folder.nullValue
case other =>
throw new IllegalArgumentException(
s"Got an unexpected value: ${other} of type: ${other.getClass.getName} from avro. This shouldn't happen...",
)
}
override def fold[B](record: GenericRecord, folder: DataFolderTo[B]): B = foldField(record, folder)
}
}
================================================
FILE: data/src/main/scala/com/thatdot/data/DataFolderTo.scala
================================================
package com.thatdot.data
import java.time._
import java.time.format.DateTimeFormatter
import scala.collection.immutable.SortedMap
import io.circe.Json
import com.thatdot.common.util.ByteConversions
trait DataFolderTo[A] {
def nullValue: A
def trueValue: A
def falseValue: A
def integer(l: Long): A
def string(s: String): A
def bytes(b: Array[Byte]): A
def floating(d: Double): A
def date(d: LocalDate): A
def time(t: OffsetTime): A
def localTime(t: LocalTime): A
def localDateTime(ldt: LocalDateTime): A
def zonedDateTime(zdt: ZonedDateTime): A
def duration(d: Duration): A
def vectorBuilder(): DataFolderTo.CollectionBuilder[A]
def mapBuilder(): DataFolderTo.MapBuilder[A]
}
object DataFolderTo {
trait CollectionBuilder[A] {
def add(a: A): Unit
def finish(): A
}
trait MapBuilder[A] {
def add(key: String, value: A): Unit
def finish(): A
}
def apply[A](implicit df: DataFolderTo[A]): DataFolderTo[A] = df
implicit val jsonFolder: DataFolderTo[Json] = new DataFolderTo[Json] {
def nullValue: Json = Json.Null
def trueValue: Json = Json.True
def falseValue: Json = Json.False
def integer(i: Long): Json = Json.fromLong(i)
def string(s: String): Json = Json.fromString(s)
def bytes(b: Array[Byte]): Json = Json.fromString(ByteConversions.formatHexBinary(b))
def floating(f: Double): Json = Json.fromDoubleOrString(f)
def date(d: LocalDate): Json = Json.fromString(d.format(DateTimeFormatter.ISO_LOCAL_DATE))
def time(t: OffsetTime): Json = Json.fromString(t.format(DateTimeFormatter.ISO_OFFSET_TIME))
def localTime(t: LocalTime): Json = Json.fromString(t.format(DateTimeFormatter.ISO_LOCAL_TIME))
def localDateTime(ldt: LocalDateTime): Json = Json.fromString(ldt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME))
def zonedDateTime(zdt: ZonedDateTime): Json = Json.fromString(zdt.format(DateTimeFormatter.ISO_ZONED_DATE_TIME))
def duration(d: Duration): Json = Json.fromString(d.toString)
def vectorBuilder(): CollectionBuilder[Json] = new CollectionBuilder[Json] {
private val elements = Vector.newBuilder[Json]
def add(a: Json): Unit = elements += a
def finish(): Json = Json.fromValues(elements.result())
}
def mapBuilder(): MapBuilder[Json] = new MapBuilder[Json] {
private val fields = Seq.newBuilder[(String, Json)]
def add(key: String, value: Json): Unit = fields += (key -> value)
def finish(): Json = Json.fromFields(fields.result())
}
}
val anyFolder: DataFolderTo[Any] = new DataFolderTo[Any] {
override def nullValue: Any = null
override def trueValue: Any = true
override def falseValue: Any = false
override def integer(l: Long): Any = l
override def string(s: String): Any = s
override def bytes(b: Array[Byte]): Any = b
override def floating(d: Double): Any = d
override def date(d: LocalDate): Any = d
override def time(t: OffsetTime): Any = t
override def localTime(t: LocalTime): Any = t
override def localDateTime(ldt: LocalDateTime): Any = ldt
override def zonedDateTime(zdt: ZonedDateTime): Any = zdt
override def duration(d: Duration): Any = d
override def vectorBuilder(): DataFolderTo.CollectionBuilder[Any] = new DataFolderTo.CollectionBuilder[Any] {
private val elements = Vector.newBuilder[Any]
def add(a: Any): Unit = elements += a
def finish(): Any = elements.result()
}
def mapBuilder(): DataFolderTo.MapBuilder[Any] = new DataFolderTo.MapBuilder[Any] {
private val kvs = SortedMap.newBuilder[String, Any]
def add(key: String, value: Any): Unit = kvs += (key -> value)
def finish(): Any = kvs.result()
}
}
}
================================================
FILE: data/src/test/scala/com/thatdot/data/AvroDecoderTest.scala
================================================
package com.thatdot.data
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import scala.collection.immutable.{SortedMap, TreeMap}
import scala.jdk.CollectionConverters._
import org.apache.avro.Schema
import org.apache.avro.generic.GenericData
import org.scalatest.funspec.AnyFunSpec
import org.scalatest.matchers.should.Matchers
class AvroDecoderTest extends AnyFunSpec with Matchers {
def canonicalize(v: Any): Any = v match {
case b: Array[_] => b.toVector
case m: Map[_, _] => m.view.mapValues(canonicalize).toMap
case m: java.util.Map[_, _] => m.asScala.view.mapValues(canonicalize).toMap
case bytes: ByteBuffer => bytes.array().toVector
case _ => v
}
it("Avro - simple types") {
val schema1 = new Schema.Parser().parse("""
|{
| "type": "record",
| "name": "testRecord",
| "fields": [
| {"name": "astring", "type": "string"},
| {"name": "anull", "type": "null"},
| {"name": "abool", "type": "boolean"},
| {"name": "aint", "type": "int"},
| {"name": "along", "type": "long"},
| {"name": "afloat", "type": "float"},
| {"name": "adouble", "type": "double"},
| {"name": "abytes", "type": "bytes"}
| ]
|}
|""".stripMargin)
val record1: GenericData.Record = new GenericData.Record(schema1)
val fieldVals = SortedMap[String, Any](
("astring" -> "string1"),
("anull" -> null),
("abool" -> true),
("aint" -> 100),
("along" -> Long.MaxValue),
("afloat" -> 101F),
("adouble" -> Double.MaxValue),
("abytes" -> ByteBuffer.wrap("some bytes".getBytes(StandardCharsets.UTF_8))),
)
fieldVals.foreach { case (s, v) => record1.put(s, v) }
val result =
DataFoldableFrom.avroDataFoldable.fold(record1, DataFolderTo.anyFolder).asInstanceOf[TreeMap[Any, Any]]
assert(canonicalize(result) == canonicalize(fieldVals))
}
it("Avro - record of records") {
val schema1 = new Schema.Parser().parse("""
|{
| "name": "multi",
| "type": "record",
| "fields": [
| {
| "name": "left",
| "type": {
| "name": "leftT",
| "type": "record",
| "fields": [ {"name": "leftA", "type": "string"}, {"name": "leftB", "type": "int"} ]
| }
| },
| {
| "name": "right",
| "type": {
| "name": "rightT",
| "type": "record",
| "fields": [ {"name": "rightA", "type": "boolean"}, {"name": "rightB", "type": "string"} ]
| }
| }
| ]
|}
|""".stripMargin)
val left: GenericData.Record = new GenericData.Record(schema1.getField("left").schema())
left.put("leftA", "a string")
left.put("leftB", 101)
val right: GenericData.Record = new GenericData.Record(schema1.getField("right").schema)
right.put("rightA", false)
right.put("rightB", "another string")
val record: GenericData.Record = new GenericData.Record(schema1)
record.put("left", left)
record.put("right", right)
val result = DataFoldableFrom.avroDataFoldable.fold(record, DataFolderTo.anyFolder)
assert(
result == TreeMap[String, TreeMap[String, Any]](
("left" -> TreeMap[String, Any](("leftA" -> "a string"), ("leftB" -> 101))),
("right" -> TreeMap[String, Any](("rightA" -> false), ("rightB" -> "another string"))),
),
)
}
it("Avro - array of maps") {
val schema1 = new Schema.Parser().parse("""
| {
| "name": "ArrayOfMaps",
| "type": "record",
| "fields": [{
| "name": "alist",
| "type": {
| "type": "array",
| "items": {
| "type": "map",
| "values": "long"
| }
| }
| }]
| }
|""".stripMargin)
val record: GenericData.Record = new GenericData.Record(schema1)
val maps: List[java.util.Map[String, Long]] = List(
Map(("k1a" -> 101L), ("k1b" -> 102L)).asJava,
Map(("k2a" -> 102L), ("k2b" -> 103L)).asJava,
)
record.put(
"alist",
new GenericData.Array[java.util.Map[String, Long]](schema1.getField("alist").schema(), maps.asJava),
)
val result = DataFoldableFrom.avroDataFoldable.fold(record, DataFolderTo.anyFolder)
assert(
canonicalize(result) == Map(
("alist" -> List(
Map(("k1a" -> 101), ("k1b" -> 102)),
Map(("k2a" -> 102), ("k2b" -> 103)),
)),
),
)
}
}
================================================
FILE: data/src/test/scala/com/thatdot/data/DataFoldableFromSpec.scala
================================================
package com.thatdot.data
import io.circe.Json
import org.scalatest.funspec.AnyFunSpec
import org.scalatest.matchers.should.Matchers
class DataFoldableFromSpec extends AnyFunSpec with Matchers {
describe("Chained foldables of the same type work") {
it("works even if types are repeated") {
val jsonValue = DataFoldableFrom.stringDataFoldable.fold("ABC", DataFolderTo.jsonFolder)
jsonValue shouldBe Json.fromString("ABC")
val jsonValue2 = DataFoldableFrom.jsonDataFoldable.fold(jsonValue, DataFolderTo.jsonFolder)
jsonValue2 shouldEqual jsonValue
}
}
}
================================================
FILE: data/src/test/scala/com/thatdot/data/DataFolderToSpec.scala
================================================
package com.thatdot.data
import org.scalatest.funspec.AnyFunSpec
import org.scalatest.matchers.should.Matchers
import com.thatdot.data.DataFoldableFrom._
class DataFolderToSpec extends AnyFunSpec with Matchers {
private def nullSafeToString(x: Any) = s"$x"
describe("DataFolderTo") {
it("preserves map values across a fold") {
val testDataStringified: Map[String, String] = FoldableTestData().asMap.view.mapValues(nullSafeToString).to(Map)
val v = stringMapDataFoldable.fold(testDataStringified, DataFolderTo.anyFolder)
v shouldBe testDataStringified
}
it("preserves vector values across a fold") {
val testDataStringified = FoldableTestData().asVector.map(nullSafeToString)
val v = stringIterableDataFoldable.fold(testDataStringified, DataFolderTo.anyFolder)
v shouldBe testDataStringified
}
}
}
================================================
FILE: data/src/test/scala/com/thatdot/data/FoldableTestData.scala
================================================
package com.thatdot.data
import java.time.{Duration => JavaDuration, LocalDate, LocalDateTime, LocalTime, OffsetTime, ZonedDateTime}
import scala.util.Random
/** Object including all types that are covered by [[DataFoldableFrom]] */
case class FoldableTestData(
nullValue: Null = null,
trueValue: Boolean = true,
falseValue: Boolean = false,
integerValue: Integer = Random.nextInt(),
stringValue: String = Random.nextString(Random.nextInt(10)),
bytesValue: Array[Byte] = Random.nextBytes(10),
floatingValue: Double = Random.nextDouble(),
dateValue: LocalDate = LocalDate.now(),
timeValue: OffsetTime = OffsetTime.now(),
localTimeValue: LocalTime = LocalTime.now(),
localDateTimeValue: LocalDateTime = LocalDateTime.now(),
zonedDateTimeValue: ZonedDateTime = ZonedDateTime.now(),
durationValue: JavaDuration = JavaDuration.ofNanos(Random.between(0L, Long.MaxValue)),
mapValue: Map[String, Any] = Map.empty[String, Any],
vectorValue: Vector[Any] = Vector.empty[Any],
) {
def asMap: Map[String, Any] =
0.until(productArity).map(i => productElementName(i) -> productElement(i)).toMap
def asVector: Vector[Any] = 0.until(productArity).map(i => productElement(i)).toVector
def foldTo[B](implicit dataFolder: DataFolderTo[B]): B = {
val mapBuilder = dataFolder.mapBuilder()
asMap.foreach { case (k, v) => mapBuilder.add(k, FoldableTestData.fromAnyDataFoldable.fold(v, dataFolder)) }
mapBuilder.finish()
}
}
object FoldableTestData {
val fromAnyDataFoldable: DataFoldableFrom[Any] = new DataFoldableFrom[Any] {
override def fold[B](value: Any, folder: DataFolderTo[B]): B =
value match {
case null => folder.nullValue
case true => folder.trueValue
case false => folder.falseValue
case s: String => folder.string(s)
case b: Array[Byte] => folder.bytes(b)
case i: Int => folder.integer(i.longValue())
case l: Long => folder.integer(l)
case d: Number => folder.floating(d.doubleValue())
case ld: LocalDate => folder.date(ld)
case ldt: LocalDateTime => folder.localDateTime(ldt)
case t: OffsetTime => folder.time(t)
case lt: LocalTime => folder.localTime(lt)
case zdt: ZonedDateTime => folder.zonedDateTime(zdt)
case dur: JavaDuration => folder.duration(dur)
case m: Map[_, _] =>
val b = folder.mapBuilder()
m.foreach { case (key, value) => b.add(key.toString, fold(value, folder)) }
b.finish()
case c: Iterable[Any] =>
val b = folder.vectorBuilder()
c.foreach(v => b.add(fold(v, folder)))
b.finish()
case other => throw new UnsupportedOperationException(s" Value $other of type ${other.getClass} is not handled")
}
}
}
================================================
FILE: model-converters/src/main/scala/com/thatdot/convert/Api2ToAws.scala
================================================
package com.thatdot.convert
import com.thatdot.{api, aws}
/** Conversions from values in the API2 model to the corresponding values in the internal AWS model. */
object Api2ToAws {
def apply(c: api.v2.AwsCredentials): aws.model.AwsCredentials =
aws.model.AwsCredentials(c.accessKeyId, c.secretAccessKey)
def apply(r: api.v2.AwsRegion): aws.model.AwsRegion = aws.model.AwsRegion(r.region)
}
================================================
FILE: model-converters/src/main/scala/com/thatdot/convert/Api2ToModel1.scala
================================================
package com.thatdot.convert
import com.thatdot.api
import com.thatdot.quine.{routes => V1}
object Api2ToModel1 {
def apply(rates: api.v2.RatesSummary): V1.RatesSummary = V1.RatesSummary(
count = rates.count,
oneMinute = rates.oneMinute,
fiveMinute = rates.fiveMinute,
fifteenMinute = rates.fifteenMinute,
overall = rates.overall,
)
def apply(c: api.v2.AwsCredentials): V1.AwsCredentials =
V1.AwsCredentials(
accessKeyId = c.accessKeyId,
secretAccessKey = c.secretAccessKey,
)
def apply(r: api.v2.AwsRegion): V1.AwsRegion = V1.AwsRegion(r.region)
}
================================================
FILE: model-converters/src/main/scala/com/thatdot/convert/Api2ToOutputs2.scala
================================================
package com.thatdot.convert
import scala.concurrent.{ExecutionContext, Future}
import org.apache.pekko.actor.ActorSystem
import com.thatdot.quine.graph.BaseGraph
import com.thatdot.quine.serialization.ProtobufSchemaCache
import com.thatdot.quine.util.StringInput
import com.thatdot.{api, outputs2}
/** Conversions from API models in [[api.v2.outputs]] to internal models in [[outputs2]]. */
object Api2ToOutputs2 {
def apply(config: api.v2.SaslJaasConfig): outputs2.SaslJaasConfig = config match {
case api.v2.PlainLogin(username, password) =>
outputs2.PlainLogin(username, password)
case api.v2.ScramLogin(username, password) =>
outputs2.ScramLogin(username, password)
case api.v2.OAuthBearerLogin(clientId, clientSecret, scope, tokenEndpointUrl) =>
outputs2.OAuthBearerLogin(clientId, clientSecret, scope, tokenEndpointUrl)
}
def apply(
format: api.v2.outputs.OutputFormat,
)(implicit protobufSchemaCache: ProtobufSchemaCache, ec: ExecutionContext): Future[outputs2.OutputEncoder] =
format match {
case api.v2.outputs.OutputFormat.JSON =>
Future.successful(outputs2.OutputEncoder.JSON())
case api.v2.outputs.OutputFormat.Protobuf(schemaUrl, typeName) =>
protobufSchemaCache
.getMessageDescriptor(StringInput.filenameOrUrl(schemaUrl), typeName, flushOnFail = true)
.map(desc => outputs2.OutputEncoder.Protobuf(schemaUrl, typeName, desc))
}
def apply(
destinationSteps: api.v2.outputs.DestinationSteps,
)(implicit
graph: BaseGraph,
ec: ExecutionContext,
protobufSchemaCache: ProtobufSchemaCache,
): Future[outputs2.FoldableDestinationSteps] = {
implicit val system: ActorSystem = graph.system
destinationSteps match {
case api.v2.outputs.DestinationSteps.Drop() =>
Future.successful(
outputs2.FoldableDestinationSteps.WithAny(
destination = outputs2.destination.Drop,
),
)
case api.v2.outputs.DestinationSteps.File(path) =>
Future.successful(
outputs2.FoldableDestinationSteps.WithByteEncoding(
// Update this when non-JSON outputs are supported for File
formatAndEncode = outputs2.OutputEncoder.JSON(),
destination = outputs2.destination.File(
path = path,
),
),
)
case api.v2.outputs.DestinationSteps.HttpEndpoint(url, parallelism, headers) =>
Future.successful(
outputs2.FoldableDestinationSteps.WithDataFoldable(
destination = outputs2.destination.HttpEndpoint(
url = url,
parallelism = parallelism,
headers = headers,
),
),
)
case api.v2.outputs.DestinationSteps.Kafka(
topic,
bootstrapServers,
format,
sslKeystorePassword,
sslTruststorePassword,
sslKeyPassword,
saslJaasConfig,
kafkaProperties,
) =>
apply(format).map(enc =>
outputs2.FoldableDestinationSteps.WithByteEncoding(
formatAndEncode = enc,
destination = outputs2.destination.Kafka(
topic = topic,
bootstrapServers = bootstrapServers,
sslKeystorePassword = sslKeystorePassword,
sslTruststorePassword = sslTruststorePassword,
sslKeyPassword = sslKeyPassword,
saslJaasConfig = saslJaasConfig.map(apply),
kafkaProperties = kafkaProperties.view.mapValues(_.s).toMap,
),
),
)
case api.v2.outputs.DestinationSteps.Kinesis(
credentials,
region,
streamName,
format,
kinesisParallelism,
kinesisMaxBatchSize,
kinesisMaxRecordsPerSecond,
kinesisMaxBytesPerSecond,
) =>
apply(format).map(enc =>
outputs2.FoldableDestinationSteps.WithByteEncoding(
formatAndEncode = enc,
destination = outputs2.destination.Kinesis(
credentials = credentials.map(Api2ToAws.apply),
region = region.map(Api2ToAws.apply),
streamName = streamName,
kinesisParallelism = kinesisParallelism,
kinesisMaxBatchSize = kinesisMaxBatchSize,
kinesisMaxRecordsPerSecond = kinesisMaxRecordsPerSecond,
kinesisMaxBytesPerSecond = kinesisMaxBytesPerSecond,
),
),
)
case api.v2.outputs.DestinationSteps.ReactiveStream(address, port, format) =>
apply(format).map(enc =>
outputs2.FoldableDestinationSteps.WithByteEncoding(
formatAndEncode = enc,
destination = outputs2.destination.ReactiveStream(
address = address,
port = port,
),
),
)
case api.v2.outputs.DestinationSteps.SNS(credentials, region, topic, format) =>
apply(format).map(enc =>
outputs2.FoldableDestinationSteps.WithByteEncoding(
formatAndEncode = enc,
destination = outputs2.destination.SNS(
credentials = credentials.map(Api2ToAws.apply),
region = region.map(Api2ToAws.apply),
topic = topic,
),
),
)
case api.v2.outputs.DestinationSteps.StandardOut() =>
Future.successful(
outputs2.FoldableDestinationSteps.WithByteEncoding(
// Update this when non-JSON outputs are supported for StandardOut
formatAndEncode = outputs2.OutputEncoder.JSON(),
destination = outputs2.destination.StandardOut,
),
)
}
}
}
================================================
FILE: model-converters/src/main/scala/com/thatdot/convert/Model1ToApi2.scala
================================================
package com.thatdot.convert
import com.thatdot.api
import com.thatdot.quine.{routes => V1}
object Model1ToApi2 {
def apply(rates: V1.RatesSummary): api.v2.RatesSummary = api.v2.RatesSummary(
count = rates.count,
oneMinute = rates.oneMinute,
fiveMinute = rates.fiveMinute,
fifteenMinute = rates.fifteenMinute,
overall = rates.overall,
)
def apply(c: V1.AwsCredentials): api.v2.AwsCredentials = api.v2.AwsCredentials(
accessKeyId = c.accessKeyId,
secretAccessKey = c.secretAccessKey,
)
def apply(r: V1.AwsRegion): api.v2.AwsRegion = api.v2.AwsRegion(r.region)
}
================================================
FILE: outputs2/src/main/scala/com/thatdot/outputs2/DestinationSteps.scala
================================================
package com.thatdot.outputs2
import org.apache.pekko.NotUsed
import org.apache.pekko.stream.scaladsl.{Flow, Sink}
import com.thatdot.common.logging.Log.LogConfig
import com.thatdot.data.DataFoldableFrom
import com.thatdot.quine.graph.NamespaceId
/** The steps that are executed to ultimately write a result to a destination
*
* Sub abstractions are:
* - For foldable destination steps [[FoldableDestinationSteps]]
* - For non-foldable destination steps [[NonFoldableDestinationSteps]]
*/
sealed trait DestinationSteps {
// TODO def post-enrichment transform
// def transform: Option[Core.PostEnrichmentTransform]
def destination: ResultDestination
}
sealed trait FoldableDestinationSteps extends DestinationSteps with DataFoldableSink {
def sink[In: DataFoldableFrom](outputName: String, namespaceId: NamespaceId)(implicit
logConfig: LogConfig,
): Sink[In, NotUsed]
}
sealed trait NonFoldableDestinationSteps extends DestinationSteps with DataNonFoldableSink
object NonFoldableDestinationSteps {
case class WithRawBytes(
destination: ResultDestination.Bytes,
) extends NonFoldableDestinationSteps {
def sink[In: BytesOutputEncoder](outputName: String, namespaceId: NamespaceId)(implicit
logConfig: LogConfig,
): Sink[In, NotUsed] =
destination.sink(outputName, namespaceId).contramap[In](implicitly[BytesOutputEncoder[In]].bytes)
}
}
object FoldableDestinationSteps {
case class WithByteEncoding(
formatAndEncode: OutputEncoder,
destination: ResultDestination.Bytes,
) extends FoldableDestinationSteps {
override def sink[In: DataFoldableFrom](outputName: String, namespaceId: NamespaceId)(implicit
logConfig: LogConfig,
): Sink[In, NotUsed] = {
val inToRepr = DataFoldableFrom[In].to(formatAndEncode.folderTo, formatAndEncode.reprTag)
val inToBytes = inToRepr.andThen(formatAndEncode.bytes)
Flow.fromFunction(inToBytes).to(destination.sink(outputName, namespaceId))
}
}
case class WithDataFoldable(destination: ResultDestination.FoldableData) extends FoldableDestinationSteps {
override def sink[In: DataFoldableFrom](outputName: String, namespaceId: NamespaceId)(implicit
logConfig: LogConfig,
): Sink[In, NotUsed] =
destination.sink(outputName, namespaceId)
}
case class WithAny(destination: ResultDestination.AnyData) extends FoldableDestinationSteps {
override def sink[In: DataFoldableFrom](outputName: String, namespaceId: NamespaceId)(implicit
logConfig: LogConfig,
): Sink[In, NotUsed] =
destination.sink(outputName, namespaceId)
}
}
================================================
FILE: outputs2/src/main/scala/com/thatdot/outputs2/OutputEncoder.scala
================================================
package com.thatdot.outputs2
import java.nio.ByteBuffer
import java.nio.charset.{Charset, StandardCharsets}
import scala.reflect.ClassTag
import com.google.protobuf.Descriptors.Descriptor
import com.thatdot.data.DataFolderTo
import com.thatdot.quine.model.QuineValue
import com.thatdot.quine.serialization.QuineValueToProtobuf
import com.thatdot.quine.serialization.data.QuineSerializationFoldersTo
sealed trait OutputEncoder {
type Repr
val reprTag: ClassTag[Repr]
def folderTo: DataFolderTo[Repr]
def bytes(value: Repr): Array[Byte]
}
sealed trait BytesOutputEncoder[Repr] {
def bytes(value: Repr): Array[Byte]
}
object BytesOutputEncoder {
def apply[A](f: A => Array[Byte]): BytesOutputEncoder[A] = new BytesOutputEncoder[A] {
override def bytes(value: A): Array[Byte] = f(value)
}
}
object OutputEncoder {
/** A JSON encoder for a [[charset]] that yields a byte array of a JSON value with a new line character appended.
*
* *NOTE* We do not currently allow the [[charset]] to be set via the API, but when we do, we will need
* to adapt [[com.thatdot.model.v2.outputs.ResultDestination.Bytes.File]] to also accommodate the `charset`
* (right now, it assumes UTF_8, since that's the default here)!
*
* @param charset the character set to use in encoding the [[io.circe.Json]] value to {{{Array[Byte]}}}
*/
case class JSON(charset: Charset = StandardCharsets.UTF_8) extends OutputEncoder {
import io.circe.{Json, Printer}
type Repr = Json
val reprTag: ClassTag[Repr] = implicitly[ClassTag[Repr]]
override def folderTo: DataFolderTo[Repr] = DataFolderTo.jsonFolder
private val printer = Printer.noSpaces
private val newline: Array[Byte] = {
val buf = charset.encode("\n")
val arr = Array.ofDim[Byte](buf.limit() - buf.position())
buf.get(arr)
arr
}
override def bytes(value: Repr): Array[Byte] = {
val buffer = printer.printToByteBuffer(value, charset)
val bufSize = buffer.limit() - buffer.position()
val arr = Array.ofDim[Byte](bufSize + newline.length)
// Add the JSON bytes to the array
buffer.get(arr, 0, bufSize)
// Add the newline bytes after the JSON bytes
ByteBuffer.wrap(newline).get(arr, bufSize, newline.length)
arr
}
}
final case class Protobuf(
schemaUrl: String,
typeName: String,
descriptor: Descriptor,
) extends OutputEncoder {
override type Repr = QuineValue
val reprTag: ClassTag[Repr] = implicitly[ClassTag[Repr]]
private val toPb: QuineValueToProtobuf = new QuineValueToProtobuf(descriptor)
override def folderTo: DataFolderTo[Repr] = QuineSerializationFoldersTo.quineValueFolder
override def bytes(value: Repr): Array[Byte] =
value match {
case QuineValue.Map(map) =>
toPb
.toProtobufBytes(map)
.fold[Array[Byte]](
failure => throw new Exception(failure.toString),
identity,
)
case _ => throw new Exception("Unable to convert a non-map to Protobuf")
}
}
}
================================================
FILE: outputs2/src/main/scala/com/thatdot/outputs2/OutputsLoggables.scala
================================================
package com.thatdot.outputs2
import com.thatdot.common.logging.Log.AlwaysSafeLoggable
object OutputsLoggables {
implicit val LogStatusCode: AlwaysSafeLoggable[org.apache.pekko.http.scaladsl.model.StatusCode] = _.value
}
================================================
FILE: outputs2/src/main/scala/com/thatdot/outputs2/ResultDestination.scala
================================================
package com.thatdot.outputs2
import com.thatdot.aws.model.{AwsCredentials, AwsRegion}
trait SinkName {
def slug: String
def sinkName(outputName: String): String = s"result-destination--$slug--$outputName"
}
/** The interface (despite the API needing an ADT) for result destinations,
* which are adapters for sending/writing to a location.
*/
sealed trait ResultDestination extends SinkName
object ResultDestination {
sealed trait Bytes extends ResultDestination with ByteArraySink
object Bytes {
trait ReactiveStream extends Bytes {
def address: String
def port: Int
}
trait StandardOut extends Bytes
trait SNS extends Bytes {
def credentials: Option[AwsCredentials]
def region: Option[AwsRegion]
def topic: String
}
trait Kafka extends Bytes {
def topic: String
def bootstrapServers: String
def kafkaProperties: Map[String, String]
}
trait Kinesis extends Bytes {
def credentials: Option[AwsCredentials]
def region: Option[AwsRegion]
def streamName: String
def kinesisParallelism: Option[Int]
def kinesisMaxBatchSize: Option[Int]
def kinesisMaxRecordsPerSecond: Option[Int]
def kinesisMaxBytesPerSecond: Option[Int]
}
trait File extends Bytes {
def path: String
}
}
sealed trait FoldableData extends ResultDestination with DataFoldableSink
object FoldableData {
trait HttpEndpoint extends FoldableData {
def url: String
def parallelism: Int
}
}
sealed trait AnyData extends ResultDestination with AnySink
object AnyData {
trait Drop extends AnyData
}
}
================================================
FILE: outputs2/src/main/scala/com/thatdot/outputs2/SaslJaasConfig.scala
================================================
package com.thatdot.outputs2
import com.thatdot.common.logging.Log.AlwaysSafeLoggable
import com.thatdot.common.security.Secret
/** Internal SASL/JAAS configuration for Kafka authentication. */
sealed trait SaslJaasConfig
object SaslJaasConfig {
/** Format a SASL/JAAS configuration as a Kafka JAAS config string.
*
* @param config
* the SASL/JAAS configuration to format
* @param renderSecret
* function to render secret values (e.g., redact or expose)
* @return
* a JAAS configuration string
*/
private def formatJaasString(config: SaslJaasConfig, renderSecret: Secret => String): String = config match {
case PlainLogin(username, password) =>
s"""org.apache.kafka.common.sec
gitextract_uxvox9yj/
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ └── feature_request.md
│ ├── PULL_REQUEST_TEMPLATE.md
│ ├── actions/
│ │ └── notify-slack-on-failure/
│ │ └── action.yml
│ └── workflows/
│ ├── ci.yml
│ ├── copy.bara.sky
│ └── copybara.yml
├── .gitignore
├── .scalafix.conf
├── .scalafmt.conf
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── api/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── api/
│ │ ├── codec/
│ │ │ └── SecretCodecs.scala
│ │ ├── schema/
│ │ │ └── SecretSchemas.scala
│ │ └── v2/
│ │ ├── ApiErrors.scala
│ │ ├── AwsCredentials.scala
│ │ ├── AwsRegion.scala
│ │ ├── RatesSummary.scala
│ │ ├── SaslJaasConfig.scala
│ │ ├── ShowShort.scala
│ │ ├── SuccessEnvelope.scala
│ │ ├── V2EndpointDefinitions.scala
│ │ ├── YamlCodec.scala
│ │ ├── codec/
│ │ │ ├── DisjointEither.scala
│ │ │ └── ThirdPartyCodecs.scala
│ │ ├── outputs/
│ │ │ ├── DestinationSteps.scala
│ │ │ ├── Format.scala
│ │ │ └── OutputFormat.scala
│ │ └── schema/
│ │ ├── TapirJsonConfig.scala
│ │ └── ThirdPartySchemas.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ ├── api/
│ │ ├── codec/
│ │ │ └── SecretCodecsSpec.scala
│ │ └── v2/
│ │ ├── ApiErrorsCodecSpec.scala
│ │ ├── AwsCredentialsCodecSpec.scala
│ │ ├── AwsGenerators.scala
│ │ ├── AwsRegionCodecSpec.scala
│ │ ├── ErrorResponseGenerators.scala
│ │ ├── ErrorTypeGenerators.scala
│ │ ├── SaslJaasConfigCodecSpec.scala
│ │ ├── SaslJaasConfigGenerators.scala
│ │ ├── SaslJaasConfigLoggableSpec.scala
│ │ ├── SuccessEnvelopeCodecSpec.scala
│ │ └── SuccessEnvelopeGenerators.scala
│ └── quine/
│ ├── JsonGenerators.scala
│ ├── ScalaPrimitiveGenerators.scala
│ └── TimeGenerators.scala
├── aws/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── aws/
│ │ ├── model/
│ │ │ ├── AwsCredentials.scala
│ │ │ └── AwsRegion.scala
│ │ └── util/
│ │ └── AwsOps.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── aws/
│ └── util/
│ └── AwsOpsSpec.scala
├── build.sbt
├── data/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── data/
│ │ ├── DataFoldableFrom.scala
│ │ └── DataFolderTo.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── data/
│ ├── AvroDecoderTest.scala
│ ├── DataFoldableFromSpec.scala
│ ├── DataFolderToSpec.scala
│ └── FoldableTestData.scala
├── model-converters/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── convert/
│ ├── Api2ToAws.scala
│ ├── Api2ToModel1.scala
│ ├── Api2ToOutputs2.scala
│ └── Model1ToApi2.scala
├── outputs2/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── outputs2/
│ │ ├── DestinationSteps.scala
│ │ ├── OutputEncoder.scala
│ │ ├── OutputsLoggables.scala
│ │ ├── ResultDestination.scala
│ │ ├── SaslJaasConfig.scala
│ │ ├── Sinks.scala
│ │ ├── destination/
│ │ │ ├── Drop.scala
│ │ │ ├── File.scala
│ │ │ ├── HttpEndpoint.scala
│ │ │ ├── Kafka.scala
│ │ │ ├── Kinesis.scala
│ │ │ ├── ReactiveStream.scala
│ │ │ ├── SNS.scala
│ │ │ └── StandardOut.scala
│ │ └── package.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── outputs2/
│ └── destination/
│ └── KafkaSpec.scala
├── project/
│ ├── Dependencies.scala
│ ├── Docker.scala
│ ├── Ecr.scala
│ ├── FlatcPlugin.scala
│ ├── GitVersion.scala
│ ├── Packaging.scala
│ ├── QuineSettings.scala
│ ├── ScalaFix.scala
│ ├── build.properties
│ ├── dependencySchemes.sbt
│ └── plugins.sbt
├── quine/
│ ├── recipes/
│ │ ├── apache_log.yaml
│ │ ├── apt-detection.yaml
│ │ ├── books.yaml
│ │ ├── cdn.yaml
│ │ ├── certstream-firehose.yaml
│ │ ├── conways-gol.yaml
│ │ ├── duration.yaml
│ │ ├── entity-resolution.yaml
│ │ ├── ethereum.yaml
│ │ ├── finance.yaml
│ │ ├── hpotter.yaml
│ │ ├── ingest.yaml
│ │ ├── kafka-ingest.yaml
│ │ ├── movieData.yaml
│ │ ├── pi.yaml
│ │ ├── ping.yaml
│ │ ├── pipe.yaml
│ │ ├── planetside-2.yaml
│ │ ├── quine-logs-recipe.yaml
│ │ ├── sq-test.yaml
│ │ ├── template-recipe.yaml
│ │ ├── webhook.yaml
│ │ ├── wikipedia-non-bot-revisions.yaml
│ │ └── wikipedia.yaml
│ └── src/
│ ├── main/
│ │ ├── resources/
│ │ │ ├── ionicons.tsv
│ │ │ ├── reference.conf
│ │ │ └── web/
│ │ │ ├── browserconfig.xml
│ │ │ ├── quine-ui-startup.js
│ │ │ ├── quine-ui.html
│ │ │ └── site.webmanifest
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── app/
│ │ ├── BaseApp.scala
│ │ ├── CmdArgs.scala
│ │ ├── ImproveQuine.scala
│ │ ├── Main.scala
│ │ ├── MeteredExecutors.scala
│ │ ├── Metrics.scala
│ │ ├── QuineApp.scala
│ │ ├── QuineAppIngestControl.scala
│ │ ├── QuinePreservingCodecs.scala
│ │ ├── Recipe.scala
│ │ ├── RecipeInterpreter.scala
│ │ ├── RecipeInterpreterV2.scala
│ │ ├── RecipePackage.scala
│ │ ├── RecipeV2.scala
│ │ ├── SchemaCache.scala
│ │ ├── StandingQueryResultOutput.scala
│ │ ├── StatusLines.scala
│ │ ├── config/
│ │ │ ├── Address.scala
│ │ │ ├── BaseConfig.scala
│ │ │ ├── EdgeIteration.scala
│ │ │ ├── FileAccessPolicy.scala
│ │ │ ├── FileIngestConfig.scala
│ │ │ ├── IdProviderType.scala
│ │ │ ├── MetricsConfig.scala
│ │ │ ├── MetricsReporter.scala
│ │ │ ├── PersistenceAgentType.scala
│ │ │ ├── PersistenceBuilder.scala
│ │ │ ├── PureconfigInstances.scala
│ │ │ ├── QuineConfig.scala
│ │ │ ├── QuinePersistenceBuilder.scala
│ │ │ ├── WebServerConfig.scala
│ │ │ └── errors/
│ │ │ └── ConfigErrorFormatter.scala
│ │ ├── data/
│ │ │ ├── QuineDataFoldablesFrom.scala
│ │ │ └── QuineDataFoldersTo.scala
│ │ ├── migrations/
│ │ │ ├── Migration.scala
│ │ │ ├── QuineMigrations.scala
│ │ │ └── instances/
│ │ │ ├── MultipleValuesRewrite.scala
│ │ │ └── package.scala
│ │ ├── model/
│ │ │ ├── README.md
│ │ │ ├── ingest/
│ │ │ │ ├── ContentDelimitedIngestSrcDef.scala
│ │ │ │ ├── IngestSrcDef.scala
│ │ │ │ ├── KafkaSrcDef.scala
│ │ │ │ ├── KinesisKclSrcDef.scala
│ │ │ │ ├── KinesisSrcDef.scala
│ │ │ │ ├── NamedPipeSource.scala
│ │ │ │ ├── ServerSentEventsSrcDef.scala
│ │ │ │ ├── SqsStreamSrcDef.scala
│ │ │ │ ├── WebsocketSimpleStartupSrcDef.scala
│ │ │ │ ├── serialization/
│ │ │ │ │ ├── ContentDecoder.scala
│ │ │ │ │ ├── CypherParseProtobuf.scala
│ │ │ │ │ ├── CypherToProtobuf.scala
│ │ │ │ │ ├── ImportFormat.scala
│ │ │ │ │ └── ProtobufParser.scala
│ │ │ │ └── util/
│ │ │ │ ├── AwsOps.scala
│ │ │ │ └── KafkaSettingsValidator.scala
│ │ │ ├── ingest2/
│ │ │ │ ├── V1IngestCodecs.scala
│ │ │ │ ├── V1IngestSchemas.scala
│ │ │ │ ├── V1ToV2.scala
│ │ │ │ ├── V2IngestEntities.scala
│ │ │ │ ├── V2IngestSources.scala
│ │ │ │ ├── V2ToV1.scala
│ │ │ │ ├── codec/
│ │ │ │ │ └── FrameDecoder.scala
│ │ │ │ ├── source/
│ │ │ │ │ ├── DecodedSource.scala
│ │ │ │ │ ├── FramedSource.scala
│ │ │ │ │ ├── IngestBounds.scala
│ │ │ │ │ └── QuineIngestQuery.scala
│ │ │ │ └── sources/
│ │ │ │ ├── CsvFileSource.scala
│ │ │ │ ├── FileSource.scala
│ │ │ │ ├── FramedSourceProvider.scala
│ │ │ │ ├── KafkaSource.scala
│ │ │ │ ├── KinesisKclSrc.scala
│ │ │ │ ├── KinesisSource.scala
│ │ │ │ ├── NumberIteratorSource.scala
│ │ │ │ ├── ReactiveSource.scala
│ │ │ │ ├── S3Source.scala
│ │ │ │ ├── ServerSentEventSource.scala
│ │ │ │ ├── SqsSource.scala
│ │ │ │ ├── StandardInputSource.scala
│ │ │ │ ├── WebSocketClientSource.scala
│ │ │ │ ├── WebSocketFileUploadSource.scala
│ │ │ │ └── package.scala
│ │ │ ├── outputs/
│ │ │ │ ├── ConsoleLoggingOutput.scala
│ │ │ │ ├── CypherQueryOutput.scala
│ │ │ │ ├── DropOutput.scala
│ │ │ │ ├── FileOutput.scala
│ │ │ │ ├── KafkaOutput.scala
│ │ │ │ ├── KinesisOutput.scala
│ │ │ │ ├── OutputRuntime.scala
│ │ │ │ ├── PostToEndpointOutput.scala
│ │ │ │ ├── QuinePatternOutput.scala
│ │ │ │ ├── SlackOutput.scala
│ │ │ │ └── SnsOutput.scala
│ │ │ ├── outputs2/
│ │ │ │ ├── QuineDestinationSteps.scala
│ │ │ │ ├── QuineResultDestination.scala
│ │ │ │ ├── destination/
│ │ │ │ │ ├── CypherQueryDestination.scala
│ │ │ │ │ └── Slack.scala
│ │ │ │ ├── package.scala
│ │ │ │ └── query/
│ │ │ │ ├── CypherQuery.scala
│ │ │ │ └── standing/
│ │ │ │ ├── Predicate.scala
│ │ │ │ ├── StandingQuery.scala
│ │ │ │ ├── StandingQueryPattern.scala
│ │ │ │ ├── StandingQueryResultTransformation.scala
│ │ │ │ ├── StandingQueryResultWorkflow.scala
│ │ │ │ ├── StandingQueryStats.scala
│ │ │ │ └── package.scala
│ │ │ └── transformation/
│ │ │ └── polyglot/
│ │ │ ├── Polyglot.scala
│ │ │ ├── PolyglotValueDataFoldableFrom.scala
│ │ │ ├── PolyglotValueDataFolderTo.scala
│ │ │ ├── Transformation.scala
│ │ │ └── langauges/
│ │ │ └── QuineJavaScript.scala
│ │ ├── routes/
│ │ │ ├── AdministrationRoutesImpl.scala
│ │ │ ├── AlgorithmRoutesImpl.scala
│ │ │ ├── BaseAppRoutes.scala
│ │ │ ├── DebugRoutesImpl.scala
│ │ │ ├── HealthAppRoutes.scala
│ │ │ ├── IngestApiMethods.scala
│ │ │ ├── IngestMeter.scala
│ │ │ ├── IngestRoutesImpl.scala
│ │ │ ├── IngestStreamState.scala
│ │ │ ├── IngestStreamWithControl.scala
│ │ │ ├── QueryUiConfigurationRoutesImpl.scala
│ │ │ ├── QueryUiConfigurationState.scala
│ │ │ ├── QueryUiCypherApiMethods.scala
│ │ │ ├── QueryUiRoutesImpl.scala
│ │ │ ├── QuineAppOpenApiDocs.scala
│ │ │ ├── QuineAppRoutes.scala
│ │ │ ├── StandingQueryInterfaceV2.scala
│ │ │ ├── StandingQueryRoutesV1Impl.scala
│ │ │ ├── StandingQueryStoreV1.scala
│ │ │ ├── Util.scala
│ │ │ ├── WebSocketQueryProtocolServer.scala
│ │ │ ├── exts/
│ │ │ │ ├── PekkoQuineEndpoints.scala
│ │ │ │ ├── ServerEntitiesWithExamples.scala
│ │ │ │ ├── ServerQuineEndpoints.scala
│ │ │ │ ├── ServerRequestTimeoutOps.scala
│ │ │ │ └── circe/
│ │ │ │ └── JsonEntitiesFromSchemas.scala
│ │ │ └── websocketquinepattern/
│ │ │ ├── LSPActor.scala
│ │ │ └── WebSocketQuinePatternServer.scala
│ │ ├── util/
│ │ │ ├── AtLeastOnceCypherQuery.scala
│ │ │ ├── OpenApiRenderer.scala
│ │ │ ├── QuineLoggables.scala
│ │ │ └── StringOps.scala
│ │ └── v2api/
│ │ ├── OssApiMethods.scala
│ │ ├── QuineOssV2OpenApiDocs.scala
│ │ ├── V2ApiInfo.scala
│ │ ├── V2OssRoutes.scala
│ │ ├── converters/
│ │ │ ├── Api2ToOutputs2.scala
│ │ │ ├── ApiToIngest.scala
│ │ │ ├── ApiToStanding.scala
│ │ │ ├── ApiToUiStyling.scala
│ │ │ ├── IngestToApi.scala
│ │ │ ├── UiStylingToApi.scala
│ │ │ └── package.scala
│ │ ├── definitions/
│ │ │ ├── AlgorithmApiMethods.scala
│ │ │ ├── ApiCommand.scala
│ │ │ ├── ApiUiStyling.scala
│ │ │ ├── CommonParameters.scala
│ │ │ ├── CypherApiMethods.scala
│ │ │ ├── DebugApiMethods.scala
│ │ │ ├── ParallelismParameter.scala
│ │ │ ├── QueryEffects.scala
│ │ │ ├── QuineApiMethods.scala
│ │ │ ├── QuineIdCodec.scala
│ │ │ ├── QuineIdSchemas.scala
│ │ │ ├── TapirDecodeErrorHandler.scala
│ │ │ ├── TapirRoutes.scala
│ │ │ ├── V2QueryExecutor.scala
│ │ │ ├── V2QueryWebSocketFlow.scala
│ │ │ ├── V2QuineEndpointDefinitions.scala
│ │ │ ├── ingest2/
│ │ │ │ ├── ApiIngest.scala
│ │ │ │ └── DeadLetterQueueOutput.scala
│ │ │ ├── outputs/
│ │ │ │ └── QuineDestinationSteps.scala
│ │ │ └── query/
│ │ │ └── standing/
│ │ │ ├── Predicate.scala
│ │ │ ├── StandingQuery.scala
│ │ │ ├── StandingQueryOutputStructure.scala
│ │ │ ├── StandingQueryPattern.scala
│ │ │ ├── StandingQueryResultTransformation.scala
│ │ │ ├── StandingQueryResultWorkflow.scala
│ │ │ └── StandingQueryStats.scala
│ │ └── endpoints/
│ │ ├── V2AlgorithmEndpoints.scala
│ │ ├── V2CypherEndpoints.scala
│ │ ├── V2DebugEndpoints.scala
│ │ ├── V2IngestEndpoints.scala
│ │ ├── V2QueryWebSocketEndpoints.scala
│ │ ├── V2QuineAdministrationEndpoints.scala
│ │ ├── V2StandingEndpoints.scala
│ │ ├── V2UiStylingEndpoints.scala
│ │ └── Visibility.scala
│ └── test/
│ ├── resources/
│ │ ├── addressbook.desc
│ │ ├── addressbook.proto
│ │ ├── application.conf
│ │ ├── documented_cassandra_config.conf
│ │ ├── documented_config.conf
│ │ ├── ingest_test_script/
│ │ │ ├── README.md
│ │ │ ├── ingest_test.py
│ │ │ └── requirements.txt
│ │ ├── multi_file_proto_test/
│ │ │ ├── README.md
│ │ │ ├── data/
│ │ │ │ ├── encode_examples.sh
│ │ │ │ ├── example_anyzone.binpb
│ │ │ │ ├── example_anyzone.txtpb
│ │ │ │ ├── example_zone_0.binpb
│ │ │ │ ├── example_zone_0.txtpb
│ │ │ │ ├── example_zone_1.binpb
│ │ │ │ ├── example_zone_1.txtpb
│ │ │ │ ├── example_zone_2.binpb
│ │ │ │ ├── example_zone_2.txtpb
│ │ │ │ ├── example_zone_3.binpb
│ │ │ │ └── example_zone_3.txtpb
│ │ │ └── schema/
│ │ │ ├── argus.proto
│ │ │ ├── azeroth.proto
│ │ │ ├── compile_schema.sh
│ │ │ ├── warcraft.desc
│ │ │ └── zone_rework.proto
│ │ ├── protobuf_test.binpb
│ │ ├── recipes/
│ │ │ ├── full.json
│ │ │ └── full.yaml
│ │ ├── trivial.cypher
│ │ └── yaml/
│ │ ├── invalid.yaml
│ │ ├── wikipedia-example.json
│ │ └── wikipedia-example.yaml
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── CirceCodecTestSupport.scala
│ ├── app/
│ │ ├── CmdArgsTest.scala
│ │ ├── ImproveQuineCodecSpec.scala
│ │ ├── ImproveQuineGenerators.scala
│ │ ├── QuineAppCodecSpec.scala
│ │ ├── QuineAppGenerators.scala
│ │ ├── QuineAppTelemetryTest.scala
│ │ ├── RecipeTest.scala
│ │ ├── RecipeV2Test.scala
│ │ ├── config/
│ │ │ ├── ClickHouseSecurityTest.scala
│ │ │ ├── ConfigGenerators.scala
│ │ │ ├── ConfigRoundTripSpec.scala
│ │ │ ├── QuineConfigTest.scala
│ │ │ ├── WebServerConfigTest.scala
│ │ │ └── errors/
│ │ │ └── ConfigErrorFormatterSpec.scala
│ │ ├── data/
│ │ │ ├── QuineDataFoldablesFromSpec.scala
│ │ │ └── QuineDataFoldersToSpec.scala
│ │ ├── ingest/
│ │ │ ├── DelimitedIngestSrcDefTest.scala
│ │ │ ├── KafkaSettingsValidatorTest.scala
│ │ │ ├── RawValuesIngestSrcDefTest.scala
│ │ │ ├── WritableInputStream.scala
│ │ │ └── serialization/
│ │ │ ├── ContentDecoderTest.scala
│ │ │ ├── CypherProtobufConversionsTest.scala
│ │ │ ├── ImportFormatTest.scala
│ │ │ └── ProtobufTest.scala
│ │ ├── model/
│ │ │ ├── ingest/
│ │ │ │ └── util/
│ │ │ │ └── AwsOpsSpec.scala
│ │ │ └── ingest2/
│ │ │ └── sources/
│ │ │ └── KafkaSourceSpec.scala
│ │ ├── routes/
│ │ │ ├── QueryUiCypherApiMethodsQuinePatternEnabledSpec.scala
│ │ │ ├── RouteHardeningOpsSpec.scala
│ │ │ └── websocketquinepattern/
│ │ │ ├── JsonRpcNotification.scala
│ │ │ ├── JsonRpcRequest.scala
│ │ │ ├── JsonRpcResponse.scala
│ │ │ └── WebSocketQuinePatternServerTest.scala
│ │ └── v2api/
│ │ └── definitions/
│ │ ├── ingest2/
│ │ │ ├── KafkaDlqSecretParamsSpec.scala
│ │ │ └── KafkaIngestSecretParamsSpec.scala
│ │ └── outputs/
│ │ └── KafkaDestinationSecretParamsSpec.scala
│ ├── convert/
│ │ └── Api2ToOutputs2KafkaSpec.scala
│ ├── graph/
│ │ ├── FakeQuineGraph.scala
│ │ └── StandingQueryTest.scala
│ ├── ingest2/
│ │ ├── IngestCodecSpec.scala
│ │ ├── IngestGenerators.scala
│ │ ├── IngestSourceTestSupport.scala
│ │ ├── V2IngestEntitiesCodecSpec.scala
│ │ ├── V2IngestEntitiesGenerators.scala
│ │ ├── V2IngestEntitiesPreservingCodecSpec.scala
│ │ ├── codec/
│ │ │ └── FrameDecoderSpec.scala
│ │ ├── source/
│ │ │ └── DecodedSourceSpec.scala
│ │ ├── sources/
│ │ │ ├── DelimitedSourcesSpec.scala
│ │ │ ├── FileLikeSourcesSpec.scala
│ │ │ ├── FramedSourceSpec.scala
│ │ │ └── KafkaFoldableSpec.scala
│ │ └── transformation/
│ │ ├── DataFoldableSpec.scala
│ │ ├── FoldableArbitraryHelpers.scala
│ │ └── QuineJavaScriptSpec.scala
│ ├── outputs/
│ │ ├── StandingQueryOutputCodecSpec.scala
│ │ └── StandingQueryOutputGenerators.scala
│ ├── routes/
│ │ ├── PostToEndpointSecretParamsSpec.scala
│ │ └── WriteToKafkaSecretParamsSpec.scala
│ └── v2api/
│ ├── ApiUiStylingCodecSpec.scala
│ ├── ApiUiStylingGenerators.scala
│ ├── EndpointValidationSpec.scala
│ ├── V2AlgorithmEndpointCodecSpec.scala
│ ├── V2AlgorithmEndpointGenerators.scala
│ ├── V2ApiCommonGenerators.scala
│ ├── V2CypherCodecSpec.scala
│ ├── V2CypherEndpointCodecSpec.scala
│ ├── V2CypherEndpointGenerators.scala
│ ├── V2DebugEndpointCodecSpec.scala
│ ├── V2DebugEndpointGenerators.scala
│ ├── V2IngestEndpointCodecSpec.scala
│ ├── V2IngestEndpointGenerators.scala
│ ├── V2QueryWebSocketFlowSpec.scala
│ ├── V2QuineAdministrationEndpointCodecSpec.scala
│ ├── V2QuineAdministrationEndpointGenerators.scala
│ ├── V2StandingEndpointCodecSpec.scala
│ └── V2StandingEndpointGenerators.scala
├── quine-browser/
│ ├── common.webpack.config.js
│ ├── dev/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── index.html
│ │ ├── package.json
│ │ ├── startup.js
│ │ ├── tsconfig.json
│ │ └── vite.config.ts
│ ├── dev.webpack.config.js
│ ├── prod.webpack.config.js
│ ├── src/
│ │ └── main/
│ │ ├── resources/
│ │ │ └── index.css
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── Util.scala
│ │ ├── routes/
│ │ │ ├── ClientRoutes.scala
│ │ │ ├── V2WebSocketQueryClient.scala
│ │ │ ├── WebSocketQueryClient.scala
│ │ │ └── exts/
│ │ │ └── ClientQuineEndpoints.scala
│ │ └── webapp/
│ │ ├── History.scala
│ │ ├── LaminarRoot.scala
│ │ ├── QuineInteractiveTS/
│ │ │ ├── _components/
│ │ │ │ ├── ConfigurationPortal.tsx
│ │ │ │ ├── IngestPortal.tsx
│ │ │ │ ├── QueryOutputPortal.tsx
│ │ │ │ ├── StandingQueryPortal.tsx
│ │ │ │ ├── _componentStyles.ts
│ │ │ │ └── index.ts
│ │ │ ├── _hooks/
│ │ │ │ └── useInterval.ts
│ │ │ ├── _services/
│ │ │ │ ├── adminService.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── ingestStreamService.ts
│ │ │ │ └── standingQueryService.ts
│ │ │ ├── _utils/
│ │ │ │ └── api.ts
│ │ │ ├── index.tsx
│ │ │ └── react.d.tsx
│ │ ├── QuineOssNavItems.scala
│ │ ├── Styles.scala
│ │ ├── Sugar.scala
│ │ ├── components/
│ │ │ ├── BoxPlot.scala
│ │ │ ├── ContextMenu.scala
│ │ │ ├── CypherResultsTable.scala
│ │ │ ├── HybridViewsRenderer.scala
│ │ │ ├── Loader.scala
│ │ │ ├── ManualHistogramPlot.scala
│ │ │ ├── PlotOrientation.scala
│ │ │ ├── Plotly.scala
│ │ │ ├── PlotlyFacade.scala
│ │ │ ├── RenderStrategy.scala
│ │ │ ├── StoplightElements.scala
│ │ │ ├── SunburstPlot.scala
│ │ │ ├── ToolbarButton.scala
│ │ │ ├── VisNetwork.scala
│ │ │ ├── dashboard/
│ │ │ │ ├── Card.scala
│ │ │ │ ├── CounterSummaryCard.scala
│ │ │ │ ├── MetricsDashboard.scala
│ │ │ │ ├── MetricsDashboardRenderer.scala
│ │ │ │ ├── ProgressBarMeter.scala
│ │ │ │ ├── ShardInfoCard.scala
│ │ │ │ └── TimerSummaryCard.scala
│ │ │ └── sidebar/
│ │ │ ├── CoreUISidebar.scala
│ │ │ ├── NavItem.scala
│ │ │ └── NavTitle.scala
│ │ ├── package.scala
│ │ ├── queryui/
│ │ │ ├── Counters.scala
│ │ │ ├── Event.scala
│ │ │ ├── GraphVisualization.scala
│ │ │ ├── HistoryNavigationButtons.scala
│ │ │ ├── MessageBar.scala
│ │ │ ├── PinTracker.scala
│ │ │ ├── QueryTypes.scala
│ │ │ ├── QueryUi.scala
│ │ │ ├── SvgSnapshot.scala
│ │ │ ├── TopBar.scala
│ │ │ └── VisNetworkVisualization.scala
│ │ ├── router/
│ │ │ ├── AppRouter.scala
│ │ │ ├── QuineOssPage.scala
│ │ │ ├── QuineOssRouter.scala
│ │ │ └── QuineOssRoutes.scala
│ │ ├── util/
│ │ │ └── LaminarUtils.scala
│ │ └── views/
│ │ ├── DocsV1View.scala
│ │ ├── DocsV2View.scala
│ │ ├── ExplorationUiView.scala
│ │ ├── MetricsView.scala
│ │ └── QuineOssViews.scala
│ └── tsconfig.json
├── quine-cassandra-persistor/
│ └── src/
│ ├── main/
│ │ ├── boilerplate/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── util/
│ │ │ └── TN.scala.template
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ └── cassandra/
│ │ ├── CassandraPersistor.scala
│ │ ├── DomainGraphNodes.scala
│ │ ├── DomainIndexEvents.scala
│ │ ├── Journals.scala
│ │ ├── MetaData.scala
│ │ ├── PrimeCassandraPersistor.scala
│ │ ├── QuinePatterns.scala
│ │ ├── Snapshots.scala
│ │ ├── StandingQueries.scala
│ │ ├── StandingQueryStates.scala
│ │ ├── aws/
│ │ │ ├── Journals.scala
│ │ │ ├── KeyspacesPersistor.scala
│ │ │ └── Snapshots.scala
│ │ ├── support/
│ │ │ ├── CassandraCodecs.scala
│ │ │ ├── CassandraColumn.scala
│ │ │ ├── CassandraStatementSettings.scala
│ │ │ ├── CassandraTable.scala
│ │ │ ├── TableDefinition.scala
│ │ │ └── syntax.scala
│ │ └── vanilla/
│ │ ├── CassandraPersistor.scala
│ │ ├── Journals.scala
│ │ └── Snapshots.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── CassandraPersistorSpec.scala
│ └── KeyspacesPersistorSpec.scala
├── quine-core/
│ └── src/
│ ├── main/
│ │ ├── fbs/
│ │ │ ├── base.fbs
│ │ │ ├── cypher.fbs
│ │ │ ├── domainindexevent.fbs
│ │ │ ├── journal.fbs
│ │ │ ├── snapshot.fbs
│ │ │ ├── standingquery.fbs
│ │ │ └── standingquerystates.fbs
│ │ ├── resources/
│ │ │ ├── logback.xml
│ │ │ ├── quine-pekko-overrides.conf
│ │ │ └── reference.conf
│ │ ├── scala/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ ├── exceptions/
│ │ │ │ ├── DuplicateIngestException.scala
│ │ │ │ ├── FileIngestSecurityException.scala
│ │ │ │ ├── JavaScriptException.scala
│ │ │ │ ├── KafkaValidationException.scala
│ │ │ │ ├── KinesisConfigurationError.scala
│ │ │ │ ├── NamespaceNotFoundException.scala
│ │ │ │ └── ShardIterationException.scala
│ │ │ ├── graph/
│ │ │ │ ├── AbstractNodeActor.scala
│ │ │ │ ├── AlgorithmGraph.scala
│ │ │ │ ├── ApiShutdownReason.scala
│ │ │ │ ├── BaseGraph.scala
│ │ │ │ ├── BaseNodeActor.scala
│ │ │ │ ├── BaseNodeActorView.scala
│ │ │ │ ├── CypherOpsGraph.scala
│ │ │ │ ├── DomainGraphNodeRegistry.scala
│ │ │ │ ├── EventTime.scala
│ │ │ │ ├── Expiration.scala
│ │ │ │ ├── GraphNodeHashCode.scala
│ │ │ │ ├── GraphNotReadyException.scala
│ │ │ │ ├── GraphQueryPattern.scala
│ │ │ │ ├── GraphService.scala
│ │ │ │ ├── GraphShardActor.scala
│ │ │ │ ├── IllegalHistoricalUpdate.scala
│ │ │ │ ├── LiteralOpsGraph.scala
│ │ │ │ ├── MasterStream.scala
│ │ │ │ ├── NamespaceSqIndex.scala
│ │ │ │ ├── NodeActor.scala
│ │ │ │ ├── NodeAndShardSupervisorStrategy.scala
│ │ │ │ ├── NodeEvent.scala
│ │ │ │ ├── NodeSnapshot.scala
│ │ │ │ ├── NodeWakeupFailedException.scala
│ │ │ │ ├── QuineIdProviders.scala
│ │ │ │ ├── QuineRuntimeFutureException.scala
│ │ │ │ ├── StandingQueryId.scala
│ │ │ │ ├── StandingQueryInfo.scala
│ │ │ │ ├── StandingQueryOpsGraph.scala
│ │ │ │ ├── StandingQueryResult.scala
│ │ │ │ ├── StaticNodeActorSupport.scala
│ │ │ │ ├── StaticNodeSupport.scala
│ │ │ │ ├── StaticShardGraph.scala
│ │ │ │ ├── WatchableEventType.scala
│ │ │ │ ├── behavior/
│ │ │ │ │ ├── ActorClock.scala
│ │ │ │ │ ├── AlgorithmBehavior.scala
│ │ │ │ │ ├── CypherBehavior.scala
│ │ │ │ │ ├── DomainNodeIndexBehavior.scala
│ │ │ │ │ ├── DomainNodeTests.scala
│ │ │ │ │ ├── GoToSleepBehavior.scala
│ │ │ │ │ ├── LiteralCommandBehavior.scala
│ │ │ │ │ ├── MultipleValuesStandingQueryBehavior.scala
│ │ │ │ │ ├── PriorityStashingBehavior.scala
│ │ │ │ │ ├── QuinePatternQueryBehavior.scala
│ │ │ │ │ └── StandingQueryBehavior.scala
│ │ │ │ ├── cypher/
│ │ │ │ │ ├── AggregationFunc.scala
│ │ │ │ │ ├── CompiledExpr.scala
│ │ │ │ │ ├── CompiledQuery.scala
│ │ │ │ │ ├── Exception.scala
│ │ │ │ │ ├── Expr.scala
│ │ │ │ │ ├── Func.scala
│ │ │ │ │ ├── Interpreter.scala
│ │ │ │ │ ├── MultipleValuesResultsReporter.scala
│ │ │ │ │ ├── MultipleValuesStandingQuery.scala
│ │ │ │ │ ├── MultipleValuesStandingQueryState.scala
│ │ │ │ │ ├── Proc.scala
│ │ │ │ │ ├── ProcedureExecutionLocation.scala
│ │ │ │ │ ├── Query.scala
│ │ │ │ │ ├── QueryContext.scala
│ │ │ │ │ ├── RunningCypherQuery.scala
│ │ │ │ │ ├── SkipOptimizingActor.scala
│ │ │ │ │ ├── Type.scala
│ │ │ │ │ ├── UserDefinedFunction.scala
│ │ │ │ │ ├── UserDefinedProcedure.scala
│ │ │ │ │ ├── VisitedVariableEdgeMatches.scala
│ │ │ │ │ └── quinepattern/
│ │ │ │ │ ├── QueryPlan.scala
│ │ │ │ │ ├── QueryPlanner.scala
│ │ │ │ │ ├── QueryStateBuilder.scala
│ │ │ │ │ ├── QueryStateHost.scala
│ │ │ │ │ ├── QuinePattern.scala
│ │ │ │ │ ├── QuinePatternExpressionInterpreter.scala
│ │ │ │ │ ├── QuinePatternFunction.scala
│ │ │ │ │ ├── QuinePatternHelpers.scala
│ │ │ │ │ └── procedures/
│ │ │ │ │ ├── GetFilteredEdgesProcedure.scala
│ │ │ │ │ ├── HelpBuiltinsProcedure.scala
│ │ │ │ │ ├── QuinePatternProcedure.scala
│ │ │ │ │ └── RecentNodesProcedure.scala
│ │ │ │ ├── edges/
│ │ │ │ │ ├── AbstractEdgeCollectionView.scala
│ │ │ │ │ ├── EdgeCollection.scala
│ │ │ │ │ ├── EdgeCollectionView.scala
│ │ │ │ │ ├── EdgeIndex.scala
│ │ │ │ │ ├── EdgeProcessor.scala
│ │ │ │ │ ├── MemoryFirstEdgeProcessor.scala
│ │ │ │ │ ├── PersistorFirstEdgeProcessor.scala
│ │ │ │ │ ├── ReverseOrderedEdgeCollection.scala
│ │ │ │ │ └── UnorderedEdgeCollection.scala
│ │ │ │ ├── messaging/
│ │ │ │ │ ├── AlgorithmMessage.scala
│ │ │ │ │ ├── BaseMessage.scala
│ │ │ │ │ ├── CypherMessage.scala
│ │ │ │ │ ├── ExactlyOnceAskActor.scala
│ │ │ │ │ ├── ExactlyOnceAskNodeActor.scala
│ │ │ │ │ ├── ExactlyOnceTimeoutException.scala
│ │ │ │ │ ├── GiveUpWaiting.scala
│ │ │ │ │ ├── LiteralMessage.scala
│ │ │ │ │ ├── LocalShardRef.scala
│ │ │ │ │ ├── NodeActorMailbox.scala
│ │ │ │ │ ├── QuineIdOps.scala
│ │ │ │ │ ├── QuineMessage.scala
│ │ │ │ │ ├── QuineRef.scala
│ │ │ │ │ ├── QuineRefOps.scala
│ │ │ │ │ ├── QuineResponse.scala
│ │ │ │ │ ├── ResultHandler.scala
│ │ │ │ │ ├── ShardActorMailbox.scala
│ │ │ │ │ ├── ShardMessage.scala
│ │ │ │ │ ├── ShardRef.scala
│ │ │ │ │ └── StandingQueryMessage.scala
│ │ │ │ ├── metrics/
│ │ │ │ │ ├── BinaryHistogramCounter.scala
│ │ │ │ │ ├── HostQuineMetrics.scala
│ │ │ │ │ └── implicits.scala
│ │ │ │ ├── package.scala
│ │ │ │ └── quinepattern/
│ │ │ │ ├── NonNodeActor.scala
│ │ │ │ ├── QuinePatternLoader.scala
│ │ │ │ ├── QuinePatternOpsGraph.scala
│ │ │ │ └── QuinePatternRegistry.scala
│ │ │ ├── migrations/
│ │ │ │ ├── MigrationError.scala
│ │ │ │ └── MigrationVersion.scala
│ │ │ ├── model/
│ │ │ │ ├── DGBOps.scala
│ │ │ │ ├── DomainGraphBranch.scala
│ │ │ │ ├── DomainGraphNode.scala
│ │ │ │ ├── DomainNodeEquiv.scala
│ │ │ │ ├── EdgeDirection.scala
│ │ │ │ ├── HalfEdge.scala
│ │ │ │ ├── Milliseconds.scala
│ │ │ │ ├── NodeComponents.scala
│ │ │ │ ├── PropertyValue.scala
│ │ │ │ ├── QuineIdProvider.scala
│ │ │ │ ├── QuineValue.scala
│ │ │ │ └── package.scala
│ │ │ ├── persistor/
│ │ │ │ ├── BinaryFormat.scala
│ │ │ │ ├── BloomFilteredPersistor.scala
│ │ │ │ ├── EmptyPersistor.scala
│ │ │ │ ├── ExceptionWrappingPersistenceAgent.scala
│ │ │ │ ├── InMemoryPersistor.scala
│ │ │ │ ├── IncompatibleVersion.scala
│ │ │ │ ├── PackedFlatBufferBinaryFormat.scala
│ │ │ │ ├── PartitionedPersistenceAgent.scala
│ │ │ │ ├── PersistenceAgent.scala
│ │ │ │ ├── PersistenceConfig.scala
│ │ │ │ ├── PrimePersistor.scala
│ │ │ │ ├── ShardedPersistor.scala
│ │ │ │ ├── StatelessPrimePersistor.scala
│ │ │ │ ├── UnifiedPrimePersistor.scala
│ │ │ │ ├── Version.scala
│ │ │ │ ├── WrappedPersistenceAgent.scala
│ │ │ │ └── codecs/
│ │ │ │ ├── DomainGraphNodeCodec.scala
│ │ │ │ ├── DomainIndexEventCodec.scala
│ │ │ │ ├── MultipleValuesStandingQueryStateCodec.scala
│ │ │ │ ├── NodeChangeEventCodec.scala
│ │ │ │ ├── PersistenceCodec.scala
│ │ │ │ ├── QueryPlanCodec.scala
│ │ │ │ ├── QuineValueCodec.scala
│ │ │ │ ├── SnapshotCodec.scala
│ │ │ │ └── StandingQueryCodec.scala
│ │ │ └── util/
│ │ │ ├── BaseError.scala
│ │ │ ├── Config.scala
│ │ │ ├── DeduplicationCache.scala
│ │ │ ├── ExpiringLruSet.scala
│ │ │ ├── Extractors.scala
│ │ │ ├── FromSingleExecutionContext.scala
│ │ │ ├── Funnels.scala
│ │ │ ├── FutureHelpers.scala
│ │ │ ├── FutureResult.scala
│ │ │ ├── GraphWithContextExt.scala
│ │ │ ├── Hashing.scala
│ │ │ ├── InterpM.scala
│ │ │ ├── Loggable.scala
│ │ │ ├── LoopbackPort.scala
│ │ │ ├── MonadHelpers.scala
│ │ │ ├── Packing.scala
│ │ │ ├── PekkoStreams.scala
│ │ │ ├── Pretty.scala
│ │ │ ├── ProgressCounter.scala
│ │ │ ├── QuineDispatchers.scala
│ │ │ ├── Retry.scala
│ │ │ ├── ReverseIterator.scala
│ │ │ ├── ReversibleLinkedHashSet.scala
│ │ │ ├── StringInput.scala
│ │ │ ├── StrongUUID.scala
│ │ │ ├── Tls.scala
│ │ │ ├── Valve.scala
│ │ │ └── ValveFlow.scala
│ │ └── scala-2.13/
│ │ └── scala/
│ │ └── compat/
│ │ └── CompatBuildFrom.scala
│ └── test/
│ ├── resources/
│ │ ├── application.conf
│ │ └── logback-test.xml
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── graph/
│ │ ├── ArbitraryInstances.scala
│ │ ├── DomainGraphNodeRegistryTest.scala
│ │ ├── EventTimeTest.scala
│ │ ├── GraphNodeHashCodeTest.scala
│ │ ├── GraphQueryPatternTest.scala
│ │ ├── HalfEdgeGen.scala
│ │ ├── HistoricalQueryTests.scala
│ │ ├── ScalaTestInstances.scala
│ │ ├── SerializationTests.scala
│ │ ├── StandingQueryResultTest.scala
│ │ ├── TestDataFactory.scala
│ │ ├── cypher/
│ │ │ ├── MultipleValuesResultsReporterTest.scala
│ │ │ └── quinepattern/
│ │ │ ├── OptionalStateCorrectnessTest.scala
│ │ │ ├── PR3981BugRegressionTest.scala
│ │ │ ├── PropertyAccessTest.scala
│ │ │ ├── QueryPlanRuntimeTest.scala
│ │ │ ├── QueryPlannerTest.scala
│ │ │ ├── StateInstallationTest.scala
│ │ │ └── TestPropertyAccess.scala
│ │ ├── edges/
│ │ │ ├── EdgeCollectionTests.scala
│ │ │ ├── ReverseOrderedEdgeCollectionTests.scala
│ │ │ ├── SyncEdgeCollectionTests.scala
│ │ │ └── UnorderedEdgeCollectionTests.scala
│ │ └── standing/
│ │ ├── AllPropertiesState.scala
│ │ ├── CrossStateTests.scala
│ │ ├── EdgeSubscriptionReciprocalStateTests.scala
│ │ ├── FilterMapStateTests.scala
│ │ ├── LabelsStateTests.scala
│ │ ├── LocalIdStateTests.scala
│ │ ├── LocalPropertyStateTests.scala
│ │ ├── StandingQueryStateHarness.scala
│ │ ├── SubscribeAcrossEdgeStateTests.scala
│ │ └── UnitSqStateTests.scala
│ ├── model/
│ │ ├── DomainGraphBranchTest.scala
│ │ └── DomainGraphNodeTest.scala
│ ├── persistor/
│ │ ├── InMemoryPersistorSpec.scala
│ │ ├── InvariantWrapper.scala
│ │ └── PersistenceAgentSpec.scala
│ ├── test/
│ │ ├── tagobjects/
│ │ │ └── IntegrationTest.scala
│ │ └── tags/
│ │ └── IntegrationTest.java
│ └── util/
│ ├── HexConversionsTest.scala
│ ├── LoggableTest.scala
│ ├── PackingTests.scala
│ ├── PrettyTests.scala
│ ├── SizeAndTimeBoundedCacheTest.scala
│ ├── StrongUUIDTest.scala
│ └── TestLogging.scala
├── quine-cypher/
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── graph/
│ │ │ └── cypher/
│ │ │ ├── CypherUDF.java
│ │ │ └── CypherUDP.java
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── bolt/
│ │ │ ├── Protocol.scala
│ │ │ ├── Serialization.scala
│ │ │ └── Structure.scala
│ │ ├── compiler/
│ │ │ └── cypher/
│ │ │ ├── CompM.scala
│ │ │ ├── Expression.scala
│ │ │ ├── Functions.scala
│ │ │ ├── GetFilteredEdges.scala
│ │ │ ├── Graph.scala
│ │ │ ├── ParametersIndex.scala
│ │ │ ├── Plan.scala
│ │ │ ├── Procedures.scala
│ │ │ ├── QueryPart.scala
│ │ │ ├── QueryScopeInfo.scala
│ │ │ ├── ReifyTime.scala
│ │ │ ├── StandingQueryPatterns.scala
│ │ │ ├── UncompiledQueryIdentity.scala
│ │ │ ├── Variables.scala
│ │ │ ├── WithFreeVariables.scala
│ │ │ ├── WithQuery.scala
│ │ │ └── package.scala
│ │ └── utils/
│ │ ├── CypherLoggables.scala
│ │ └── MonadVia.scala
│ └── test/
│ ├── resources/
│ │ └── application.conf
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── Bolt.scala
│ ├── BoltSerializations.scala
│ ├── CypherAggregations.scala
│ ├── CypherEquality.scala
│ ├── CypherErrors.scala
│ ├── CypherExpressions.scala
│ ├── CypherFunctions.scala
│ ├── CypherLists.scala
│ ├── CypherMatch.scala
│ ├── CypherMatchPerformance.scala
│ ├── CypherMatrix.scala
│ ├── CypherMerge.scala
│ ├── CypherRecursiveSubQuery.scala
│ ├── CypherReturn.scala
│ ├── CypherShortestPath.scala
│ ├── CypherStrings.scala
│ ├── CypherSubQueries.scala
│ ├── CypherTemporal.scala
│ ├── QueryStaticTest.scala
│ ├── SkipOptimizationsTest.scala
│ ├── VariableLengthRelationshipPattern.scala
│ └── compiler/
│ └── cypher/
│ ├── CypherComplete.scala
│ ├── CypherHarness.scala
│ ├── CypherMutate.scala
│ ├── HistoricalQueryTests.scala
│ ├── OrderedEdgesTest.scala
│ ├── SkipUninterestingNodesTest.scala
│ └── StandingQueryPatternsTest.scala
├── quine-docs/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── docs/
│ │ ├── GenerateCypherTables.scala
│ │ ├── GenerateOpenApi.scala
│ │ ├── GenerateOpenApiV2.scala
│ │ └── GenerateRecipeDirectory.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── docs/
│ ├── GenerateOpenApiTest.scala
│ └── GenerateOpenApiTestV2.scala
├── quine-endpoints/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── routes/
│ │ │ ├── AdministrationRoutes.scala
│ │ │ ├── AlgorithmRoutes.scala
│ │ │ ├── DebugOpsRoutes.scala
│ │ │ ├── IngestRoutes.scala
│ │ │ ├── QueryProtocol.scala
│ │ │ ├── QueryUiConfigurationRoutes.scala
│ │ │ ├── QueryUiRoutes.scala
│ │ │ ├── QuickQuery.scala
│ │ │ ├── StandingQueryRoutes.scala
│ │ │ └── exts/
│ │ │ ├── AnySchema.scala
│ │ │ ├── EndpointsWithCustomErrorText.scala
│ │ │ ├── EntitiesWithExamples.scala
│ │ │ └── QuineEndpoints.scala
│ │ └── v2api/
│ │ └── routes/
│ │ ├── V2MetricsRoutes.scala
│ │ ├── V2QueryUiConfigurationRoutes.scala
│ │ └── V2QueryUiRoutes.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── routes/
│ └── AwsSchemaSpec.scala
├── quine-endpoints2/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── api/
│ └── v2/
│ ├── QueryWebSocketProtocol.scala
│ ├── TapirCirceUnifiedConfig.scala
│ └── TypeDiscriminatorConfig.scala
├── quine-gremlin/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── gremlin/
│ │ ├── Exceptions.scala
│ │ ├── GremlinLexer.scala
│ │ ├── GremlinParser.scala
│ │ ├── GremlinQueryRunner.scala
│ │ ├── GremlinTypes.scala
│ │ ├── GremlinValue.scala
│ │ └── package.scala
│ └── test/
│ ├── application.conf
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── gremlin/
│ ├── ErrorMessages.scala
│ ├── GremlinHarness.scala
│ └── SimpleQueries.scala
├── quine-language/
│ └── src/
│ ├── main/
│ │ ├── antlr4/
│ │ │ └── Cypher.g4
│ │ ├── java/
│ │ │ └── com/
│ │ │ └── thatdot/
│ │ │ └── quine/
│ │ │ └── language/
│ │ │ ├── server/
│ │ │ │ ├── QuineLanguageServer.java
│ │ │ │ └── QuineTextDocumentService.java
│ │ │ └── testclient/
│ │ │ └── QuineLanguageClient.java
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ ├── cypher/
│ │ │ ├── CollectingErrorListener.scala
│ │ │ ├── ast/
│ │ │ │ └── AST.scala
│ │ │ ├── phases/
│ │ │ │ ├── LexerPhase.scala
│ │ │ │ ├── MaterializationPhase.scala
│ │ │ │ ├── ParserPhase.scala
│ │ │ │ └── SymbolAnalysis.scala
│ │ │ ├── utils/
│ │ │ │ └── Helpers.scala
│ │ │ └── visitors/
│ │ │ ├── ast/
│ │ │ │ ├── CreateVisitor.scala
│ │ │ │ ├── EffectVisitor.scala
│ │ │ │ ├── ForeachVisitor.scala
│ │ │ │ ├── InQueryCallVisitor.scala
│ │ │ │ ├── MatchClauseVisitor.scala
│ │ │ │ ├── MultiPartQueryVisitor.scala
│ │ │ │ ├── NodeLabelVisitor.scala
│ │ │ │ ├── PatternVisitor.scala
│ │ │ │ ├── ProjectionBodyVisitor.scala
│ │ │ │ ├── ProjectionItemVisitor.scala
│ │ │ │ ├── QueryVisitor.scala
│ │ │ │ ├── ReadingClauseVisitor.scala
│ │ │ │ ├── RegularQueryVisitor.scala
│ │ │ │ ├── ReturnVisitor.scala
│ │ │ │ ├── SetItemVisitor.scala
│ │ │ │ ├── SinglePartQueryVisitor.scala
│ │ │ │ ├── SingleQueryVisitor.scala
│ │ │ │ ├── UnwindClauseVisitor.scala
│ │ │ │ ├── UpdatingClauseVisitor.scala
│ │ │ │ ├── WhereClauseVisitor.scala
│ │ │ │ ├── WithVisitor.scala
│ │ │ │ ├── expressions/
│ │ │ │ │ ├── AddSubtractVisitor.scala
│ │ │ │ │ ├── AndVisitor.scala
│ │ │ │ │ ├── AtomVisitor.scala
│ │ │ │ │ ├── ComparisonVisitor.scala
│ │ │ │ │ ├── DoubleVisitor.scala
│ │ │ │ │ ├── ExpressionVisitor.scala
│ │ │ │ │ ├── FunctionInvocationVisitor.scala
│ │ │ │ │ ├── IntegerVisitor.scala
│ │ │ │ │ ├── LiteralVisitor.scala
│ │ │ │ │ ├── MapLiteralVisitor.scala
│ │ │ │ │ ├── MultiplyDivideModuloVisitor.scala
│ │ │ │ │ ├── NonArithmeticOperatorVisitor.scala
│ │ │ │ │ ├── NumberVisitor.scala
│ │ │ │ │ ├── OrExpressionVisitor.scala
│ │ │ │ │ ├── ParameterVisitor.scala
│ │ │ │ │ ├── PartialComparisonVisitor.scala
│ │ │ │ │ ├── PowerOfVisitor.scala
│ │ │ │ │ ├── PropertyVisitor.scala
│ │ │ │ │ ├── StringListNullVisitor.scala
│ │ │ │ │ ├── UnaryAddSubtractVisitor.scala
│ │ │ │ │ ├── VariableVisitor.scala
│ │ │ │ │ └── XorVisitor.scala
│ │ │ │ └── patterns/
│ │ │ │ ├── AnonymousPatternPartVisitor.scala
│ │ │ │ ├── MatchPatternVisitor.scala
│ │ │ │ ├── NodePatternVisitor.scala
│ │ │ │ ├── PatternElementChainVisitor.scala
│ │ │ │ ├── PatternElementVisitor.scala
│ │ │ │ ├── PatternExpVisitor.scala
│ │ │ │ ├── PatternPartVisitor.scala
│ │ │ │ └── RelationshipPatternVisitor.scala
│ │ │ └── semantic/
│ │ │ ├── AddSubtractVisitor.scala
│ │ │ ├── AndVisitor.scala
│ │ │ ├── AnonymousPatternPartVisitor.scala
│ │ │ ├── AtomVisitor.scala
│ │ │ ├── ComparisonVisitor.scala
│ │ │ ├── CreateVisitor.scala
│ │ │ ├── DoubleVisitor.scala
│ │ │ ├── ExpressionVisitor.scala
│ │ │ ├── FunctionInvocationVisitor.scala
│ │ │ ├── InQueryCallVisitor.scala
│ │ │ ├── IntegerVisitor.scala
│ │ │ ├── LiteralVisitor.scala
│ │ │ ├── MapLiteralVisitor.scala
│ │ │ ├── MatchClauseVisitor.scala
│ │ │ ├── MultiPartQueryVisitor.scala
│ │ │ ├── MultiplyDivideModuloVisitor.scala
│ │ │ ├── NodeLabelVisitor.scala
│ │ │ ├── NodePatternVisitor.scala
│ │ │ ├── NonArithmeticOperatorVisitor.scala
│ │ │ ├── NotVisitor.scala
│ │ │ ├── NumberVisitor.scala
│ │ │ ├── OrExpressionVisitor.scala
│ │ │ ├── ParameterVisitor.scala
│ │ │ ├── PartialComparisonVisitor.scala
│ │ │ ├── PatternElementChainVisitor.scala
│ │ │ ├── PatternElementVisitor.scala
│ │ │ ├── PatternPartVisitor.scala
│ │ │ ├── PatternVisitor.scala
│ │ │ ├── PowerOfVisitor.scala
│ │ │ ├── ProjectionBodyVisitor.scala
│ │ │ ├── ProjectionItemVisitor.scala
│ │ │ ├── PropertyExpressionVisitor.scala
│ │ │ ├── QueryVisitor.scala
│ │ │ ├── ReadingClauseVisitor.scala
│ │ │ ├── RegularQueryVisitor.scala
│ │ │ ├── RelationshipPatternVisitor.scala
│ │ │ ├── ReturnVisitor.scala
│ │ │ ├── SetItemVisitor.scala
│ │ │ ├── SinglePartQueryVisitor.scala
│ │ │ ├── SingleQueryVisitor.scala
│ │ │ ├── StringListNullVisitor.scala
│ │ │ ├── UnaryAddOrSubtractVisitor.scala
│ │ │ ├── UnwindClauseVisitor.scala
│ │ │ ├── UpdatingClauseVisitor.scala
│ │ │ ├── VariableVisitor.scala
│ │ │ ├── WhereVisitor.scala
│ │ │ └── XorVisitor.scala
│ │ └── language/
│ │ ├── Cypher.scala
│ │ ├── ast/
│ │ │ └── AST.scala
│ │ ├── diagnostic/
│ │ │ └── Diagnostic.scala
│ │ ├── domain/
│ │ │ └── Graph.scala
│ │ ├── phases/
│ │ │ ├── CompilerPhase.scala
│ │ │ ├── CompilerState.scala
│ │ │ ├── Phase.scala
│ │ │ ├── TypeCheckingPhase.scala
│ │ │ └── Upgrade.scala
│ │ ├── prettyprint/
│ │ │ ├── ASTInstances.scala
│ │ │ ├── BaseInstances.scala
│ │ │ ├── CypherASTInstances.scala
│ │ │ ├── PrettyPrint.scala
│ │ │ ├── ResultInstances.scala
│ │ │ ├── SymbolAnalysisInstances.scala
│ │ │ ├── TypeInstances.scala
│ │ │ └── package.scala
│ │ ├── semantic/
│ │ │ └── Semantics.scala
│ │ ├── server/
│ │ │ ├── ContextAwareLanguageService.scala
│ │ │ ├── Helpers.scala
│ │ │ └── SimpleTrie.scala
│ │ ├── testclient/
│ │ │ ├── QuineLanguageClient.scala
│ │ │ └── TestProgram.scala
│ │ └── types/
│ │ └── Type.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ ├── cypher/
│ │ ├── phases/
│ │ │ ├── LexerPhaseTest.scala
│ │ │ ├── ParserPhaseTest.scala
│ │ │ └── PhaseCompositionTest.scala
│ │ └── visitors/
│ │ └── ast/
│ │ ├── AddSubtractVisitorTests.scala
│ │ ├── AndVisitorTests.scala
│ │ ├── AtomVisitorTests.scala
│ │ ├── ComparisonVisitorTests.scala
│ │ ├── DoubleVisitorTests.scala
│ │ ├── ExpressionVisitorTests.scala
│ │ ├── IntegerVisitorTests.scala
│ │ ├── LiteralVisitorTests.scala
│ │ ├── MapLiteralVisitorTests.scala
│ │ ├── MultiplyDivideModuloVisitorTests.scala
│ │ ├── NonArithmeticOperatorVisitorTests.scala
│ │ ├── NumberVisitorTests.scala
│ │ ├── OrExpressionVisitorTests.scala
│ │ ├── ParameterVisitorTests.scala
│ │ ├── PartialComparisonVisitorTests.scala
│ │ ├── PowerOfVisitorTests.scala
│ │ ├── PropertyVisitorTests.scala
│ │ ├── UnaryAddSubtractVisitorTests.scala
│ │ └── XorVisitorTests.scala
│ └── language/
│ ├── diagnostic/
│ │ └── DiagnosticTest.scala
│ ├── parser/
│ │ └── ParserTests.scala
│ ├── phases/
│ │ ├── AlphaRenamingTests.scala
│ │ ├── MaterializationTests.scala
│ │ ├── PipelineExplorer.scala
│ │ └── SymbolAnalysisTests.scala
│ ├── prettyprint/
│ │ └── PrettyPrintTest.scala
│ ├── semantics/
│ │ └── SemanticAnalysisTests.scala
│ ├── server/
│ │ ├── ContextAwareLanguageServiceTest.scala
│ │ ├── LanguageServerHelper.scala
│ │ ├── QuineLanguageServerTest.scala
│ │ ├── QuineTextDocumentServiceTest.scala
│ │ └── TextDocumentServiceHelper.scala
│ └── types/
│ ├── GraphElementTypeTests.scala
│ ├── TypeCheckerTests.scala
│ ├── TypeEntryDuplicateTest.scala
│ └── TypeSystemTest.scala
├── quine-mapdb-persistor/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ ├── MapDbGlobalPersistor.scala
│ │ └── MapDbPersistor.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── MapDbPersistorSpec.scala
│ └── MapDbPersistorTests.scala
├── quine-rocksdb-persistor/
│ └── src/
│ ├── main/
│ │ └── scala/
│ │ └── com/
│ │ └── thatdot/
│ │ └── quine/
│ │ └── persistor/
│ │ ├── RocksDbPersistor.scala
│ │ └── RocksDbPrimePersistor.scala
│ └── test/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── persistor/
│ ├── RocksDbKeyEncodingTest.scala
│ ├── RocksDbPersistorSpec.scala
│ └── RocksDbPersistorTests.scala
├── quine-serialization/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── quine/
│ └── serialization/
│ ├── AvroSchemaCache.scala
│ ├── EncoderDecoder.scala
│ ├── ProtobufSchemaCache.scala
│ ├── QuineValueToProtobuf.scala
│ ├── SchemaError.scala
│ └── data/
│ ├── QuineSerializationFoldablesFrom.scala
│ └── QuineSerializationFoldersTo.scala
├── visnetwork-facade/
│ └── src/
│ └── main/
│ └── scala/
│ └── com/
│ └── thatdot/
│ └── visnetwork/
│ ├── DataSet.scala
│ ├── Events.scala
│ ├── Network.scala
│ └── package.scala
└── vite-shared/
├── base.config.ts
├── fixtures/
│ ├── metrics.ts
│ ├── query-results.ts
│ └── ui-config.ts
├── index.ts
├── package.json
├── plugins/
│ ├── mock-api-factory.ts
│ └── serve-scalajs-bundle.ts
├── tsconfig.json
└── utils/
└── mime-types.ts
SYMBOL INDEX (102 symbols across 19 files)
FILE: quine-browser/dev/startup.js
function parseMillis (line 5) | function parseMillis(atTime) {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_components/IngestPortal.tsx
type IngestStreamProps (line 15) | type IngestStreamProps = {
type CreateModalProps (line 21) | type CreateModalProps = {
type DetailsModalProps (line 26) | type DetailsModalProps = {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_components/QueryOutputPortal.tsx
type QueryOutputProps (line 13) | type QueryOutputProps = {
type CreateModalProps (line 20) | type CreateModalProps = {
type DetailsModalProps (line 25) | type DetailsModalProps = {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_components/StandingQueryPortal.tsx
type StandingQueryProps (line 13) | type StandingQueryProps = {
type CreateModalProps (line 18) | type CreateModalProps = {
type DetailsModalProps (line 23) | type DetailsModalProps = {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_hooks/useInterval.ts
type Delay (line 3) | type Delay = number | null;
type TimerHandler (line 4) | type TimerHandler = (...args: any[]) => void;
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_services/adminService.ts
function getQuineConfiguration (line 3) | function getQuineConfiguration() {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_services/ingestStreamService.ts
function getAllIngestStreams (line 3) | function getAllIngestStreams() {
function createIngestStream (line 11) | function createIngestStream(name: string, body:object) {
function cancelIngestStream (line 20) | function cancelIngestStream(name: string) {
function pauseIngestStream (line 28) | function pauseIngestStream(name: string) {
function startIngestStream (line 36) | function startIngestStream(name: string) {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_services/standingQueryService.ts
function getAllStandingQueries (line 3) | function getAllStandingQueries() {
function createStandingQuery (line 11) | function createStandingQuery(name: string, body:object) {
function cancelStandingQuery (line 20) | function cancelStandingQuery(name: string) {
function registerQueryOutput (line 28) | function registerQueryOutput(standingQueryName: string, outputName: stri...
function cancelQueryOutput (line 37) | function cancelQueryOutput(standingQueryName: string, outputName: string) {
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/_utils/api.ts
function handleResponse (line 11) | async function handleResponse(response: { headers: { get: (arg0: string)...
function request (line 35) | async function request({ path, opts = {}, rootURL = '' } : {path:string,...
function get (line 53) | async function get({ path, parameters = {}, opts = {} } : {path:string, ...
function post (line 71) | async function post({ path, body = {}, opts = {} } : {path:string, body?...
function put (line 89) | async function put({ path, body = {}, opts = {} } : {path:string, body?:...
function del (line 107) | async function del({ path, body = {}, opts = {} } : {path:string, body?:...
FILE: quine-browser/src/main/scala/com/thatdot/quine/webapp/QuineInteractiveTS/index.tsx
constant WS_URL (line 31) | const WS_URL = "ws://0.0.0.0:8080/api/v1/query"
FILE: quine-language/src/main/java/com/thatdot/quine/language/server/QuineLanguageServer.java
class QuineLanguageServer (line 16) | public class QuineLanguageServer implements LanguageServer, LanguageClie...
method QuineLanguageServer (line 21) | public QuineLanguageServer() {
method initialize (line 25) | @Override
method connect (line 55) | @Override
method shutdown (line 60) | @Override
method exit (line 65) | @Override
method getTextDocumentService (line 68) | @Override
method getWorkspaceService (line 73) | @Override
method setTrace (line 78) | public void setTrace(SetTraceParams params) {}
FILE: quine-language/src/main/java/com/thatdot/quine/language/server/QuineTextDocumentService.java
class QuineTextDocumentService (line 16) | public class QuineTextDocumentService implements TextDocumentService {
method QuineTextDocumentService (line 21) | public QuineTextDocumentService(ContextAwareLanguageService cals) {
method getTextDocument (line 27) | public TextDocumentItem getTextDocument(String uri) {
method didOpen (line 31) | @Override
method didChange (line 37) | @Override
method didClose (line 45) | @Override
method didSave (line 50) | @Override
method completion (line 54) | @Override
method diagnostic (line 68) | @Override
method semanticTokensRange (line 95) | @Override
method semanticTokensFull (line 102) | @Override
FILE: quine-language/src/main/java/com/thatdot/quine/language/testclient/QuineLanguageClient.java
class QuineLanguageClient (line 11) | public class QuineLanguageClient implements LanguageClient {
method telemetryEvent (line 12) | @Override
method publishDiagnostics (line 15) | @Override
method showMessage (line 18) | @Override
method showMessageRequest (line 21) | @Override
method logMessage (line 27) | @Override
FILE: quine/src/main/resources/web/quine-ui-startup.js
function parseMillis (line 2) | function parseMillis(atTime) {
function deriveProxySafeBaseURI (line 26) | function deriveProxySafeBaseURI() {
FILE: quine/src/test/resources/ingest_test_script/ingest_test.py
class Encoding (line 22) | class Encoding:
method parse_csv (line 25) | def parse_csv(cls, encoding_csv: str):
method encode_value (line 33) | def encode_value(cls, encoding: str, value: Any):
method decode_value (line 42) | def decode_value(cls, encoding: str, value: Any):
method encode (line 51) | def encode(cls, encodings: List[str], value: str) ->str:
method decode (line 58) | def decode(cls, encodings: List[str], value: str) ->str:
function random_string (line 64) | def random_string(ct: int = 10):
class TestConfig (line 68) | class TestConfig:
method __init__ (line 70) | def __init__(self, name:str, count: int, quine_url: str, encodings: Li...
method recipe (line 75) | def recipe(self):
method generate_values (line 78) | def generate_values(self):
method write_values (line 82) | def write_values(self, values: List[Any]) -> None:
method create_recipe (line 85) | def create_recipe(self):
method retrieve_values (line 91) | def retrieve_values(self):
method get_ingested_ct (line 96) | def get_ingested_ct(self):
method run_test (line 100) | def run_test(self, sleep_time_ms=20000, write=True, read = True):
method req (line 123) | def req(self, method: str, path: str, **kwargs) -> Optional[Response]:
class KinesisConfig (line 141) | class KinesisConfig(TestConfig):
method __init__ (line 143) | def __init__(self, name: str, count: int, quine_url: str, stream_name:...
method recipe (line 149) | def recipe(self):
method write_values (line 163) | def write_values(self, values: List[str]):
class SQSConfig (line 168) | class SQSConfig(TestConfig):
method __init__ (line 169) | def __init__(self, name: str,count: int, quine_url: str, queue_url: st...
method recipe (line 174) | def recipe(self):
method write_values (line 183) | def write_values(self, values: List[str]) -> None:
class KafkaConfig (line 195) | class KafkaConfig(TestConfig):
method __init__ (line 197) | def __init__(self, name: str,count: int, quine_url: str, topic: str, k...
method recipe (line 205) | def recipe(self):
method write_values (line 214) | def write_values(self, values: List[str]):
FILE: vite-shared/base.config.ts
type ScalaJSProjectConfig (line 5) | interface ScalaJSProjectConfig {
function createBaseConfig (line 60) | function createBaseConfig(config: ScalaJSProjectConfig): UserConfig {
FILE: vite-shared/plugins/mock-api-factory.ts
type MockApiHandler (line 4) | type MockApiHandler = (
type MockApiHandlerMap (line 10) | type MockApiHandlerMap = Record<string, MockApiHandler>;
type MockApiOptions (line 12) | interface MockApiOptions {
function respondJson (line 42) | function respondJson(res: ServerResponse, data: unknown, status = 200): ...
function wrapV2Response (line 53) | function wrapV2Response(content: unknown): unknown {
function parseRequestBody (line 64) | function parseRequestBody(req: IncomingMessage): Promise<unknown> {
function createBaseHandlers (line 84) | function createBaseHandlers(fixtures: MockApiOptions['fixtures']): MockA...
function createOpenApiHandlers (line 181) | function createOpenApiHandlers(productName: string): MockApiHandlerMap {
function createMockApiPlugin (line 222) | function createMockApiPlugin(options: MockApiOptions): Plugin {
FILE: vite-shared/plugins/serve-scalajs-bundle.ts
type ScalaJSBundleOptions (line 6) | interface ScalaJSBundleOptions {
function createScalaJSBundlePlugin (line 35) | function createScalaJSBundlePlugin(options: ScalaJSBundleOptions): Plugin {
FILE: vite-shared/utils/mime-types.ts
constant MIME_TYPES (line 6) | const MIME_TYPES: Record<string, string> = {
function getMimeType (line 50) | function getMimeType(filePath: string): string {
Condensed preview — 1062 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (7,269K chars).
[
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 834,
"preview": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the b"
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 595,
"preview": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your fea"
},
{
"path": ".github/PULL_REQUEST_TEMPLATE.md",
"chars": 1380,
"preview": "# Description\n\nPlease include a summary of the change and which issue is fixed. Please also include relevant motivation "
},
{
"path": ".github/actions/notify-slack-on-failure/action.yml",
"chars": 1136,
"preview": "name: Notify Slack on CI Failure\ndescription: Posts a failure notification to Slack via incoming webhook\n\ninputs:\n job-"
},
{
"path": ".github/workflows/ci.yml",
"chars": 1858,
"preview": "name: CI\n\nconcurrency:\n group: ci-${{ github.head_ref }}\n cancel-in-progress: true\n\non:\n push:\n branches:\n - "
},
{
"path": ".github/workflows/copy.bara.sky",
"chars": 4150,
"preview": "SOT_REPO = \"git@github.com:thatdot/quine-plus.git\"\nSOT_BRANCH = \"main\"\nDESTINATION_REPO = \"git@github.com:thatdot/quine."
},
{
"path": ".github/workflows/copybara.yml",
"chars": 2576,
"preview": "name: Copy Commits to thatdot/quine Repo\n\non:\n push:\n branches:\n - main\n pull_request_target:\n workflow_dispa"
},
{
"path": ".gitignore",
"chars": 983,
"preview": "*.db\n*.data\n*.class\n\n# likely binaries\nquine-*.jar\nnovelty-*.jar\n\n# local settings\n.java-version\n.jvmopts\nlocal.sbt\nloca"
},
{
"path": ".scalafix.conf",
"chars": 1521,
"preview": "// .scalafix.conf\nrules = [\n OrganizeImports\n ExplicitResultTypes\n LeakingImplicitClassVal\n DisableSyntax\n// \"githu"
},
{
"path": ".scalafmt.conf",
"chars": 798,
"preview": "version = 2.7.5 // scala-steward:off\n\n// Additional style conventions not enforced by scalafmt:\n// - mark `case class`es"
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 3083,
"preview": "# Code of Conduct\n\n## Code of Conduct for the Quine Community\n\nthatDot is dedicated to providing the best community poss"
},
{
"path": "CONTRIBUTING.md",
"chars": 808,
"preview": "# Contributing\n\nThe community is the heart of all open-source projects. We welcome contributions from all people and str"
},
{
"path": "LICENSE",
"chars": 2019,
"preview": "MIT License with Commons Clause\n\nCopyright © 2014 Ryan Wright; © 2019 thatDot, Inc.\n\nPermission is hereby granted, free "
},
{
"path": "README.md",
"chars": 5639,
"preview": "\n[![GitHub Release](https://img.shields.io/git"
},
{
"path": "api/src/main/scala/com/thatdot/api/codec/SecretCodecs.scala",
"chars": 1068,
"preview": "package com.thatdot.api.codec\n\nimport io.circe.{Decoder, Encoder}\n\nimport com.thatdot.common.security.Secret\n\n/** Circe "
},
{
"path": "api/src/main/scala/com/thatdot/api/schema/SecretSchemas.scala",
"chars": 503,
"preview": "package com.thatdot.api.schema\n\nimport sttp.tapir.Schema\n\nimport com.thatdot.common.security.Secret\n\n/** Tapir schemas f"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/ApiErrors.scala",
"chars": 10551,
"preview": "package com.thatdot.api.v2\n\nimport java.util.UUID\n\nimport io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, der"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/AwsCredentials.scala",
"chars": 1948,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}\ni"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/AwsRegion.scala",
"chars": 816,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.{Decoder, Encoder}\nimport sttp.tapir.Schema\nimport sttp.tapir.Schema.annotat"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/RatesSummary.scala",
"chars": 1078,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}\ni"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/SaslJaasConfig.scala",
"chars": 7139,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}\ni"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/ShowShort.scala",
"chars": 673,
"preview": "package com.thatdot.api.v2\n\ntrait ShowShort[-A] {\n def showShort(a: A): String\n}\n\ntrait ShowShortOps {\n implicit class"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/SuccessEnvelope.scala",
"chars": 2211,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.generic.extras.Configuration\nimport io.circe.generic.extras.semiauto.{derive"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/V2EndpointDefinitions.scala",
"chars": 9763,
"preview": "package com.thatdot.api.v2\n\nimport java.nio.charset.{Charset, StandardCharsets}\nimport java.util.concurrent.TimeUnit\n\nim"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/YamlCodec.scala",
"chars": 932,
"preview": "package com.thatdot.api.v2\n\nimport io.circe._\nimport io.circe.syntax._\nimport sttp.model.MediaType\nimport sttp.tapir.{Co"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/codec/DisjointEither.scala",
"chars": 3365,
"preview": "package com.thatdot.api.v2.codec\n\nimport io.circe.{Decoder, Encoder}\n\n/** Evidence that A and B are structurally disjoin"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/codec/ThirdPartyCodecs.scala",
"chars": 925,
"preview": "package com.thatdot.api.v2.codec\n\nimport java.nio.charset.Charset\nimport java.time.Instant\n\nimport scala.util.Try\n\nimpor"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/outputs/DestinationSteps.scala",
"chars": 7143,
"preview": "package com.thatdot.api.v2.outputs\n\nimport io.circe.{Decoder, Encoder}\n\nimport com.thatdot.api.v2.{AwsCredentials, AwsRe"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/outputs/Format.scala",
"chars": 80,
"preview": "package com.thatdot.api.v2.outputs\n\ntrait Format {\n val format: OutputFormat\n}\n"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/outputs/OutputFormat.scala",
"chars": 1418,
"preview": "package com.thatdot.api.v2.outputs\n\nimport io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEn"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/schema/TapirJsonConfig.scala",
"chars": 556,
"preview": "package com.thatdot.api.v2.schema\n\nimport io.circe.Printer\nimport sttp.tapir.json.circe.TapirJsonCirce\n\n/** Provides `js"
},
{
"path": "api/src/main/scala/com/thatdot/api/v2/schema/ThirdPartySchemas.scala",
"chars": 2168,
"preview": "package com.thatdot.api.v2.schema\n\nimport java.nio.charset.Charset\nimport java.time.Instant\n\nimport scala.util.{Failure,"
},
{
"path": "api/src/test/scala/com/thatdot/api/codec/SecretCodecsSpec.scala",
"chars": 1710,
"preview": "package com.thatdot.api.codec\n\nimport io.circe.Json\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.matchers.shou"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/ApiErrorsCodecSpec.scala",
"chars": 3469,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.funsuite.AnyFunSuite\nimport org.scala"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/AwsCredentialsCodecSpec.scala",
"chars": 3813,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.Json\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.matchers.should."
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/AwsGenerators.scala",
"chars": 1148,
"preview": "package com.thatdot.api.v2\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport com.thatdot.common.security.Secret\nimport com"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/AwsRegionCodecSpec.scala",
"chars": 1000,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.Json\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.funsuite.AnyFunS"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/ErrorResponseGenerators.scala",
"chars": 1380,
"preview": "package com.thatdot.api.v2\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport com.thatdot.quine.ScalaPrimitiveGenerators\n\no"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/ErrorTypeGenerators.scala",
"chars": 1097,
"preview": "package com.thatdot.api.v2\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport com.thatdot.quine.ScalaPrimitiveGenerators\n\no"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigCodecSpec.scala",
"chars": 10454,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.funsuite.AnyFunSuite\nimport org.scala"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigGenerators.scala",
"chars": 2140,
"preview": "package com.thatdot.api.v2\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport com.thatdot.common.security.Secret\nimport com"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/SaslJaasConfigLoggableSpec.scala",
"chars": 3459,
"preview": "package com.thatdot.api.v2\n\nimport org.scalatest.funsuite.AnyFunSuite\nimport org.scalatest.matchers.should.Matchers\n\nimp"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/SuccessEnvelopeCodecSpec.scala",
"chars": 5403,
"preview": "package com.thatdot.api.v2\n\nimport io.circe.syntax.EncoderOps\nimport org.scalatest.funsuite.AnyFunSuite\nimport org.scala"
},
{
"path": "api/src/test/scala/com/thatdot/api/v2/SuccessEnvelopeGenerators.scala",
"chars": 1384,
"preview": "package com.thatdot.api.v2\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nimport com.thatdot.quine.ScalaPrimitiveGenerators\n\no"
},
{
"path": "api/src/test/scala/com/thatdot/quine/JsonGenerators.scala",
"chars": 1152,
"preview": "package com.thatdot.quine\n\nimport io.circe.Json\nimport org.scalacheck.{Arbitrary, Gen}\n\nobject JsonGenerators {\n import"
},
{
"path": "api/src/test/scala/com/thatdot/quine/ScalaPrimitiveGenerators.scala",
"chars": 1708,
"preview": "package com.thatdot.quine\n\nimport org.scalacheck.{Arbitrary, Gen}\n\n/** Popular primitive-based generators (no `Arbs`; wo"
},
{
"path": "api/src/test/scala/com/thatdot/quine/TimeGenerators.scala",
"chars": 1347,
"preview": "package com.thatdot.quine\n\nimport java.time.Instant\n\nimport org.scalacheck.{Arbitrary, Gen}\n\nobject TimeGenerators {\n o"
},
{
"path": "aws/src/main/scala/com/thatdot/aws/model/AwsCredentials.scala",
"chars": 152,
"preview": "package com.thatdot.aws.model\n\nimport com.thatdot.common.security.Secret\n\nfinal case class AwsCredentials(accessKeyId: S"
},
{
"path": "aws/src/main/scala/com/thatdot/aws/model/AwsRegion.scala",
"chars": 74,
"preview": "package com.thatdot.aws.model\n\nfinal case class AwsRegion(region: String)\n"
},
{
"path": "aws/src/main/scala/com/thatdot/aws/util/AwsOps.scala",
"chars": 3206,
"preview": "package com.thatdot.aws.util\n\nimport scala.reflect.{ClassTag, classTag}\n\nimport software.amazon.awssdk.auth.credentials."
},
{
"path": "aws/src/test/scala/com/thatdot/aws/util/AwsOpsSpec.scala",
"chars": 1816,
"preview": "package com.thatdot.aws.util\n\nimport org.scalatest.matchers.should.Matchers\nimport org.scalatest.wordspec.AnyWordSpec\nim"
},
{
"path": "build.sbt",
"chars": 24122,
"preview": "import Dependencies.*\nimport scalajsbundler.util.JSON._\nimport QuineSettings.*\n\nThisBuild / resolvers += \"thatDot maven\""
},
{
"path": "data/src/main/scala/com/thatdot/data/DataFoldableFrom.scala",
"chars": 11138,
"preview": "package com.thatdot.data\n\nimport scala.collection.{SeqView, View, mutable}\nimport scala.jdk.CollectionConverters._\nimpor"
},
{
"path": "data/src/main/scala/com/thatdot/data/DataFolderTo.scala",
"chars": 3751,
"preview": "package com.thatdot.data\n\nimport java.time._\nimport java.time.format.DateTimeFormatter\n\nimport scala.collection.immutabl"
},
{
"path": "data/src/test/scala/com/thatdot/data/AvroDecoderTest.scala",
"chars": 4705,
"preview": "package com.thatdot.data\n\nimport java.nio.ByteBuffer\nimport java.nio.charset.StandardCharsets\n\nimport scala.collection.i"
},
{
"path": "data/src/test/scala/com/thatdot/data/DataFoldableFromSpec.scala",
"chars": 589,
"preview": "package com.thatdot.data\n\nimport io.circe.Json\nimport org.scalatest.funspec.AnyFunSpec\nimport org.scalatest.matchers.sho"
},
{
"path": "data/src/test/scala/com/thatdot/data/DataFolderToSpec.scala",
"chars": 862,
"preview": "package com.thatdot.data\n\nimport org.scalatest.funspec.AnyFunSpec\nimport org.scalatest.matchers.should.Matchers\n\nimport "
},
{
"path": "data/src/test/scala/com/thatdot/data/FoldableTestData.scala",
"chars": 2787,
"preview": "package com.thatdot.data\n\nimport java.time.{Duration => JavaDuration, LocalDate, LocalDateTime, LocalTime, OffsetTime, Z"
},
{
"path": "model-converters/src/main/scala/com/thatdot/convert/Api2ToAws.scala",
"chars": 401,
"preview": "package com.thatdot.convert\n\nimport com.thatdot.{api, aws}\n\n/** Conversions from values in the API2 model to the corresp"
},
{
"path": "model-converters/src/main/scala/com/thatdot/convert/Api2ToModel1.scala",
"chars": 601,
"preview": "package com.thatdot.convert\n\nimport com.thatdot.api\nimport com.thatdot.quine.{routes => V1}\n\nobject Api2ToModel1 {\n\n de"
},
{
"path": "model-converters/src/main/scala/com/thatdot/convert/Api2ToOutputs2.scala",
"chars": 5755,
"preview": "package com.thatdot.convert\n\nimport scala.concurrent.{ExecutionContext, Future}\n\nimport org.apache.pekko.actor.ActorSyst"
},
{
"path": "model-converters/src/main/scala/com/thatdot/convert/Model1ToApi2.scala",
"chars": 603,
"preview": "package com.thatdot.convert\n\nimport com.thatdot.api\nimport com.thatdot.quine.{routes => V1}\n\nobject Model1ToApi2 {\n\n de"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/DestinationSteps.scala",
"chars": 2605,
"preview": "package com.thatdot.outputs2\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.stream.scaladsl.{Flow, Sink}\n\nimpo"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/OutputEncoder.scala",
"chars": 3100,
"preview": "package com.thatdot.outputs2\n\nimport java.nio.ByteBuffer\nimport java.nio.charset.{Charset, StandardCharsets}\n\nimport sca"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/OutputsLoggables.scala",
"chars": 224,
"preview": "package com.thatdot.outputs2\n\nimport com.thatdot.common.logging.Log.AlwaysSafeLoggable\n\nobject OutputsLoggables {\n impl"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/ResultDestination.scala",
"chars": 1654,
"preview": "package com.thatdot.outputs2\n\nimport com.thatdot.aws.model.{AwsCredentials, AwsRegion}\n\ntrait SinkName {\n def slug: Str"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/SaslJaasConfig.scala",
"chars": 2444,
"preview": "package com.thatdot.outputs2\n\nimport com.thatdot.common.logging.Log.AlwaysSafeLoggable\nimport com.thatdot.common.securit"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/Sinks.scala",
"chars": 831,
"preview": "package com.thatdot.outputs2\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.stream.scaladsl.Sink\n\nimport com.t"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/Drop.scala",
"chars": 541,
"preview": "package com.thatdot.outputs2.destination\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.stream.scaladsl.Sink\n\n"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/File.scala",
"chars": 874,
"preview": "package com.thatdot.outputs2.destination\n\nimport java.nio.file.{Paths, StandardOpenOption}\n\nimport org.apache.pekko.NotU"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/HttpEndpoint.scala",
"chars": 3371,
"preview": "package com.thatdot.outputs2.destination\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure,"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/Kafka.scala",
"chars": 3426,
"preview": "package com.thatdot.outputs2.destination\n\nimport scala.annotation.unused\n\nimport org.apache.pekko.NotUsed\nimport org.apa"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/Kinesis.scala",
"chars": 2589,
"preview": "package com.thatdot.outputs2.destination\n\nimport scala.util.{Failure, Random, Success}\n\nimport org.apache.pekko.NotUsed\n"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/ReactiveStream.scala",
"chars": 2087,
"preview": "package com.thatdot.outputs2.destination\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.actor.ActorSystem\nimpo"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/SNS.scala",
"chars": 2035,
"preview": "package com.thatdot.outputs2.destination\n\nimport scala.util.{Failure, Success}\n\nimport org.apache.pekko.NotUsed\nimport o"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/destination/StandardOut.scala",
"chars": 622,
"preview": "package com.thatdot.outputs2.destination\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.stream.scaladsl.Sink\n\n"
},
{
"path": "outputs2/src/main/scala/com/thatdot/outputs2/package.scala",
"chars": 132,
"preview": "package com.thatdot\n\n/** The Outputs V2 definitions. These must be and remain available to all products. */\npackage obje"
},
{
"path": "outputs2/src/test/scala/com/thatdot/outputs2/destination/KafkaSpec.scala",
"chars": 4234,
"preview": "package com.thatdot.outputs2.destination\n\nimport org.apache.pekko.actor.ActorSystem\n\nimport org.scalatest.BeforeAndAfter"
},
{
"path": "project/Dependencies.scala",
"chars": 4835,
"preview": "import sbt._\n\nobject Dependencies {\n val amazonKinesisClientV = \"3.4.2\"\n val apacheCommonsCsvV = \"1.14.1\"\n val avroV "
},
{
"path": "project/Docker.scala",
"chars": 5454,
"preview": "import scala.concurrent.duration.*\nimport scala.sys.process.*\n\nimport sbt.*\nimport sbt.Keys.{baseDirectory, name, stream"
},
{
"path": "project/Ecr.scala",
"chars": 1848,
"preview": "import java.nio.charset.StandardCharsets.UTF_8\nimport sbt._\nimport sbt.Keys.streams\nimport sbtdocker.DockerKeys.{docker,"
},
{
"path": "project/FlatcPlugin.scala",
"chars": 4366,
"preview": "import sbt._\nimport sbt.Keys._\nimport sbt.util.CacheImplicits._\n\nimport scala.util.Properties\n\nobject FlatcPlugin extend"
},
{
"path": "project/GitVersion.scala",
"chars": 878,
"preview": "import sbt.{AutoPlugin, SettingKey}\nimport sbt.Keys.version\nimport com.github.sbt.git.SbtGit.GitKeys.gitReader\nimport co"
},
{
"path": "project/Packaging.scala",
"chars": 4108,
"preview": "import sbtassembly.{Assembly, AssemblyPlugin, CustomMergeStrategy, MergeStrategy, PathList}\nimport sbtassembly.AssemblyK"
},
{
"path": "project/QuineSettings.scala",
"chars": 4537,
"preview": "import sbt._\nimport sbt.Keys._\nimport org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._\nimport scalajsbu"
},
{
"path": "project/ScalaFix.scala",
"chars": 609,
"preview": "import sbt._\nimport sbt.Keys.{semanticdbEnabled, semanticdbVersion}\nimport scalafix.sbt.ScalafixPlugin\n\n// Extra scalafi"
},
{
"path": "project/build.properties",
"chars": 18,
"preview": "sbt.version=1.12.9"
},
{
"path": "project/dependencySchemes.sbt",
"chars": 360,
"preview": "// scala-xml should be compatible across 1.x and 2.x. Dependencies of the meta-build itself require\n// conflicting major"
},
{
"path": "project/plugins.sbt",
"chars": 1504,
"preview": "// resolvers += \"Typesafe repository\" at \"http://repo.typesafe.com/typesafe/releases/\"\nval scalajsBundlerVersion = \"0.21"
},
{
"path": "quine/recipes/apache_log.yaml",
"chars": 1518,
"preview": "version: 1\ntitle: Apache Log Analytics\ncontributor: https://github.com/joshcody\nsummary: ''\ndescription: ''\ningestStream"
},
{
"path": "quine/recipes/apt-detection.yaml",
"chars": 8844,
"preview": "title: APT Detection\nsummary: Endpoint logs and network traffic data merge to auto-detect exfiltration\ncontributor: http"
},
{
"path": "quine/recipes/books.yaml",
"chars": 4275,
"preview": "version: 1\ntitle: Book ratings demo\ncontributor: https://github.com/stevenbenjamin\nsummary: Demonstration of building a "
},
{
"path": "quine/recipes/cdn.yaml",
"chars": 32500,
"preview": "version: 1\ntitle: CDN Cache Efficiency By Segment\ncontributor: https://www.linkedin.com/in/alokaggarwal2\nsummary:\n Real"
},
{
"path": "quine/recipes/certstream-firehose.yaml",
"chars": 933,
"preview": "version: 1\ntitle: Certstream Firehose\ncontributor: https://github.com/emanb29\nsummary: Log new SSL certificate registrat"
},
{
"path": "quine/recipes/conways-gol.yaml",
"chars": 10885,
"preview": "version: 1\ntitle: Conway's Game of Life\ncontributor: Matthew Cullum https://github.com/brackishman\nsummary: Conway's Gam"
},
{
"path": "quine/recipes/duration.yaml",
"chars": 4508,
"preview": "version: 1\ntitle: Temporal Locality Example\ncontributor: https://github.com/maglietti\nsummary: Relate email messages sen"
},
{
"path": "quine/recipes/entity-resolution.yaml",
"chars": 12498,
"preview": "version: 1\ntitle: Entity Resolution Example\ncontributor: https://github.com/rrwright\nsummary: Entity Resolution\ndescript"
},
{
"path": "quine/recipes/ethereum.yaml",
"chars": 7687,
"preview": "version: 1\ntitle: Ethereum Tag Propagation\ncontributor: https://github.com/emanb29\nsummary: Ethereum Blockchain model wi"
},
{
"path": "quine/recipes/finance.yaml",
"chars": 11472,
"preview": "version: 1\ntitle: Financial Risk Recipe\ndescription: |-\n The financial industry’s current approach to managing mandated"
},
{
"path": "quine/recipes/hpotter.yaml",
"chars": 1359,
"preview": "version: 1\ntitle: Harry Potter\ncontributor: https://github.com/harpocrates\nsummary: Small graph of connected nodes\ndescr"
},
{
"path": "quine/recipes/ingest.yaml",
"chars": 463,
"preview": "version: 1\ntitle: Ingest\ncontributor: https://github.com/landon9720\nsummary: Ingest input file lines as graph nodes\ndesc"
},
{
"path": "quine/recipes/kafka-ingest.yaml",
"chars": 519,
"preview": "version: 1\ntitle: Kafka Ingest\ncontributor: https://github.com/landon9720\nsummary: Ingest Kafka topic messages as graph "
},
{
"path": "quine/recipes/movieData.yaml",
"chars": 7173,
"preview": "# Recipe schema version (currently only supported value is 1;)\nversion: 1\n\n# Identifies the Recipe but is not necessaril"
},
{
"path": "quine/recipes/pi.yaml",
"chars": 3218,
"preview": "version: 1\ntitle: Pi\ncontributor: https://github.com/emanb29\nsummary: Incrementally approximates pi using Leibniz' formu"
},
{
"path": "quine/recipes/ping.yaml",
"chars": 859,
"preview": "version: 1\ntitle: Ping\ncontributor: https://github.com/landon9720\nsummary: Ingest input file lines and echo to output fi"
},
{
"path": "quine/recipes/pipe.yaml",
"chars": 845,
"preview": "version: 1\ntitle: Pipe\ncontributor: https://github.com/landon9720\nsummary: Ingest from Standard Input and writes to Stan"
},
{
"path": "quine/recipes/planetside-2.yaml",
"chars": 4275,
"preview": "version: 1\ntitle: Planetside 2\ncontributor: https://github.com/emanb29\nsummary: Models real-time player kill data from P"
},
{
"path": "quine/recipes/quine-logs-recipe.yaml",
"chars": 4786,
"preview": "version: 1\ntitle: Quine Log Reader\ncontributor: https://github.com/maglietti\nsummary: \"Ingest Quine Log Lines\"\ndescripti"
},
{
"path": "quine/recipes/sq-test.yaml",
"chars": 3035,
"preview": "version: 1\ntitle: Standing Query Test Recipe\ncontributor: https://github.com/rrwright\nsummary: Create a mathematically d"
},
{
"path": "quine/recipes/template-recipe.yaml",
"chars": 1563,
"preview": "# Recipe schema version (currently only supported value is 1; 🎉)\nversion: 1\n\n# Identifies the Recipe but is not necessar"
},
{
"path": "quine/recipes/webhook.yaml",
"chars": 1685,
"preview": "version: 1\ntitle: Data Enrichment with Webhooks\ncontributor: https://github.com/mastapegs\nsummary: Stream numbers into g"
},
{
"path": "quine/recipes/wikipedia-non-bot-revisions.yaml",
"chars": 2433,
"preview": "version: 1\ntitle: Wikipedia non-bot page update event stream\ncontributor: https://github.com/thatdot\nsummary: Stream pag"
},
{
"path": "quine/recipes/wikipedia.yaml",
"chars": 2684,
"preview": "version: 1\ntitle: Ingest Wikipedia Page Create stream\ncontributor: https://github.com/landon9720\nsummary: Consume events"
},
{
"path": "quine/src/main/resources/ionicons.tsv",
"chars": 14837,
"preview": "ion-alert\t\nion-alert-circled\t\nion-android-add\t\nion-android-add-circle\t\nion-android-alarm-clock\t\nion-android-alert\t"
},
{
"path": "quine/src/main/resources/reference.conf",
"chars": 1273,
"preview": "include classpath(\"quine-pekko-overrides\")\n\npekko {\n // This timeout controls the browsers timeout when waiting for API"
},
{
"path": "quine/src/main/resources/web/browserconfig.xml",
"chars": 246,
"preview": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<browserconfig>\n <msapplication>\n <tile>\n <square150x150logo"
},
{
"path": "quine/src/main/resources/web/quine-ui-startup.js",
"chars": 2294,
"preview": "// Given some value meant to represent time, return either integer milliseconds or undefined\nfunction parseMillis(atTime"
},
{
"path": "quine/src/main/resources/web/quine-ui.html",
"chars": 1216,
"preview": "<!DOCTYPE html>\n\n<html style=\"height: 100%\">\n<head>\n <title>Quine</title>\n <meta charset=\"utf-8\">\n\n <link rel=\"apple-"
},
{
"path": "quine/src/main/resources/web/site.webmanifest",
"chars": 426,
"preview": "{\n \"name\": \"\",\n \"short_name\": \"\",\n \"icons\": [\n {\n \"src\": \"/android-chrome-192x192.png\",\n "
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/BaseApp.scala",
"chars": 13208,
"preview": "package com.thatdot.quine.app\n\nimport java.nio.charset.StandardCharsets.UTF_8\n\nimport scala.concurrent.{ExecutionContext"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/CmdArgs.scala",
"chars": 3524,
"preview": "package com.thatdot.quine.app\n\nimport scopt.OEffect._\nimport scopt.OParser\n\n/** Data model for Quine command line progra"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/ImproveQuine.scala",
"chars": 10956,
"preview": "package com.thatdot.quine.app\n\nimport java.net.NetworkInterface\nimport java.nio.ByteBuffer\nimport java.nio.charset.Stand"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/Main.scala",
"chars": 15439,
"preview": "package com.thatdot.quine.app\n\nimport java.io.File\nimport java.net.URL\nimport java.nio.charset.{Charset, StandardCharset"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/MeteredExecutors.scala",
"chars": 8160,
"preview": "package com.thatdot.quine.app\n\nimport java.util.concurrent.{ExecutorService, ThreadFactory}\n\nimport org.apache.pekko.dis"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/Metrics.scala",
"chars": 1049,
"preview": "package com.thatdot.quine.app\n\nimport java.lang.management.ManagementFactory\n\nimport scala.collection.mutable.ListBuffer"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/QuineApp.scala",
"chars": 77269,
"preview": "package com.thatdot.quine.app\n\nimport java.time.Instant\nimport java.time.temporal.ChronoUnit.MILLIS\nimport java.util.UUI"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/QuineAppIngestControl.scala",
"chars": 1587,
"preview": "package com.thatdot.quine.app\n\nimport scala.concurrent.{ExecutionContext, Future}\n\nimport org.apache.pekko.Done\nimport o"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/QuinePreservingCodecs.scala",
"chars": 1471,
"preview": "package com.thatdot.quine.app\n\nimport io.circe.Encoder\n\nimport com.thatdot.common.security.Secret\nimport com.thatdot.qui"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/Recipe.scala",
"chars": 27411,
"preview": "package com.thatdot.quine.app\n\nimport java.io.{File, FileNotFoundException}\nimport java.net.HttpURLConnection.{HTTP_MOVE"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/RecipeInterpreter.scala",
"chars": 12694,
"preview": "package com.thatdot.quine.app\n\nimport java.lang.System.lineSeparator\nimport java.net.URL\nimport java.util.concurrent.Tim"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/RecipeInterpreterV2.scala",
"chars": 14317,
"preview": "package com.thatdot.quine.app\n\nimport java.net.URL\nimport java.util.concurrent.TimeoutException\n\nimport scala.concurrent"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/RecipePackage.scala",
"chars": 1430,
"preview": "package com.thatdot.quine.app\n\nimport java.nio.file.{Files, Path}\n\nimport io.circe\n\n/** Container for a Recipe that also"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/RecipeV2.scala",
"chars": 15167,
"preview": "package com.thatdot.quine.app\n\nimport cats.data.{NonEmptyList, ValidatedNel}\nimport cats.implicits._\nimport io.circe.gen"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/SchemaCache.scala",
"chars": 218,
"preview": "package com.thatdot.quine.app\n\nimport com.thatdot.quine.serialization.{AvroSchemaCache, ProtobufSchemaCache}\n\ntrait Sche"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/StandingQueryResultOutput.scala",
"chars": 14869,
"preview": "package com.thatdot.quine.app\n\nimport java.util.concurrent.atomic.AtomicReference\n\nimport scala.concurrent.Future\nimport"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/StatusLines.scala",
"chars": 3444,
"preview": "package com.thatdot.quine.app\n\nimport java.io.PrintStream\n\nimport scala.collection.mutable\nimport scala.concurrent.block"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/Address.scala",
"chars": 587,
"preview": "package com.thatdot.quine.app.config\n\nimport java.net.InetSocketAddress\n\nimport com.google.common.net.HostAndPort\n\nobjec"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/BaseConfig.scala",
"chars": 3074,
"preview": "package com.thatdot.quine.app.config\n\nimport java.nio.file.{Files, Path}\n\nimport cats.syntax.either._\nimport com.typesaf"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/EdgeIteration.scala",
"chars": 928,
"preview": "package com.thatdot.quine.app.config\n\nimport pureconfig.ConfigConvert\nimport pureconfig.generic.semiauto.deriveEnumerati"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/FileAccessPolicy.scala",
"chars": 7151,
"preview": "package com.thatdot.quine.app.config\n\nimport java.nio.file.{Files, Path, Paths}\n\nimport cats.data.ValidatedNel\nimport ca"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/FileIngestConfig.scala",
"chars": 2367,
"preview": "package com.thatdot.quine.app.config\n\nimport pureconfig.error.CannotConvert\nimport pureconfig.generic.semiauto.deriveCon"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/IdProviderType.scala",
"chars": 2807,
"preview": "package com.thatdot.quine.app.config\n\nimport java.{util => ju}\n\nimport memeid.{UUID => UUID4s}\nimport pureconfig.ConfigC"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/MetricsConfig.scala",
"chars": 326,
"preview": "package com.thatdot.quine.app.config\n\nimport pureconfig.ConfigConvert\nimport pureconfig.generic.semiauto.deriveConvert\n\n"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/MetricsReporter.scala",
"chars": 4249,
"preview": "package com.thatdot.quine.app.config\n\nimport java.io.File\n\nimport scala.concurrent.duration.FiniteDuration\nimport scala."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/PersistenceAgentType.scala",
"chars": 7536,
"preview": "package com.thatdot.quine.app.config\n\nimport java.io.File\nimport java.net.InetSocketAddress\nimport java.nio.file.Paths\n\n"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/PersistenceBuilder.scala",
"chars": 9141,
"preview": "package com.thatdot.quine.app.config\n\nimport java.io.File\nimport java.util.Properties\n\nimport scala.concurrent.Await\nimp"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/PureconfigInstances.scala",
"chars": 2607,
"preview": "package com.thatdot.quine.app.config\n\nimport scala.concurrent.duration.FiniteDuration\nimport scala.jdk.CollectionConvert"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/QuineConfig.scala",
"chars": 2905,
"preview": "package com.thatdot.quine.app.config\n\nimport scala.concurrent.duration.{Duration, DurationInt, FiniteDuration}\n\nimport o"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/QuinePersistenceBuilder.scala",
"chars": 752,
"preview": "package com.thatdot.quine.app.config\n\nimport java.io.File\n\n/** Persistence builder instance for Quine.\n *\n * Uses Quin"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/WebServerConfig.scala",
"chars": 3170,
"preview": "package com.thatdot.quine.app.config\n\nimport java.io.File\nimport java.net.{InetAddress, URL}\n\nimport org.apache.pekko.ht"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/config/errors/ConfigErrorFormatter.scala",
"chars": 8780,
"preview": "package com.thatdot.quine.app.config.errors\n\nimport cats.data.NonEmptyList\nimport pureconfig.error.{ConfigReaderFailure,"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/data/QuineDataFoldablesFrom.scala",
"chars": 2550,
"preview": "package com.thatdot.quine.app.data\n\nimport com.thatdot.data.{DataFoldableFrom, DataFolderTo}\nimport com.thatdot.quine.gr"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/data/QuineDataFoldersTo.scala",
"chars": 1845,
"preview": "package com.thatdot.quine.app.data\n\nimport java.time.{Duration, LocalDate, LocalDateTime, LocalTime, OffsetTime, ZonedDa"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/migrations/Migration.scala",
"chars": 1755,
"preview": "package com.thatdot.quine.app.migrations\n\nimport scala.concurrent.Future\n\nimport com.thatdot.quine.migrations.{Migration"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/migrations/QuineMigrations.scala",
"chars": 7246,
"preview": "package com.thatdot.quine.app.migrations\n\nimport scala.concurrent.{ExecutionContext, Future}\n\nimport com.thatdot.common."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/migrations/instances/MultipleValuesRewrite.scala",
"chars": 320,
"preview": "package com.thatdot.quine.app.migrations.instances\n\nimport com.thatdot.quine.app.migrations.Migration\nimport com.thatdot"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/migrations/instances/package.scala",
"chars": 509,
"preview": "package com.thatdot.quine.app.migrations\n\n/** This package contains an object for each feature that may require an out-o"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/README.md",
"chars": 368,
"preview": "## Models\n\nThis package contains internal object models. They implement the effects described in the user facing API.\n\nW"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/ContentDelimitedIngestSrcDef.scala",
"chars": 14730,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport scala.util.Success\n\nimport org.apache.pekko.NotUsed\nimport org.apache"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/IngestSrcDef.scala",
"chars": 21518,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport java.nio.charset.{Charset, StandardCharsets}\n\nimport scala.concurrent"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/KafkaSrcDef.scala",
"chars": 13919,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport scala.concurrent.duration.{Duration, FiniteDuration, MILLISECONDS}\nim"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/KinesisKclSrcDef.scala",
"chars": 20749,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport java.net.InetAddress\nimport java.nio.ByteBuffer\nimport java.util.{Cal"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/KinesisSrcDef.scala",
"chars": 6958,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport java.time.Instant\n\nimport scala.collection.Set\nimport scala.concurren"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/NamedPipeSource.scala",
"chars": 5893,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport java.nio.ByteBuffer\nimport java.nio.channels.FileChannel\nimport java."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/ServerSentEventsSrcDef.scala",
"chars": 1499,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.http.scaladsl.Http\ni"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/SqsStreamSrcDef.scala",
"chars": 4029,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport scala.concurrent.Future\nimport scala.util.{Success, Try}\n\nimport org."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/WebsocketSimpleStartupSrcDef.scala",
"chars": 4718,
"preview": "package com.thatdot.quine.app.model.ingest\n\nimport scala.concurrent.duration.DurationInt\nimport scala.concurrent.{Execut"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/serialization/ContentDecoder.scala",
"chars": 3915,
"preview": "package com.thatdot.quine.app.model.ingest.serialization\n\nimport java.io.{ByteArrayInputStream, ByteArrayOutputStream}\ni"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/serialization/CypherParseProtobuf.scala",
"chars": 3316,
"preview": "package com.thatdot.quine.app.model.ingest.serialization\n\nimport java.net.URL\n\nimport scala.util.Try\n\nimport org.apache."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/serialization/CypherToProtobuf.scala",
"chars": 2512,
"preview": "package com.thatdot.quine.app.model.ingest.serialization\n\nimport java.net.URL\n\nimport scala.util.Try\n\nimport org.apache."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/serialization/ImportFormat.scala",
"chars": 10311,
"preview": "package com.thatdot.quine.app.model.ingest.serialization\n\nimport scala.concurrent.{ExecutionContext, Future, Promise}\nim"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/serialization/ProtobufParser.scala",
"chars": 748,
"preview": "package com.thatdot.quine.app.model.ingest.serialization\n\nimport com.google.protobuf.Descriptors.Descriptor\nimport com.g"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/util/AwsOps.scala",
"chars": 3323,
"preview": "package com.thatdot.quine.app.model.ingest.util\n\nimport scala.reflect.{ClassTag, classTag}\n\nimport software.amazon.awssd"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest/util/KafkaSettingsValidator.scala",
"chars": 12560,
"preview": "package com.thatdot.quine.app.model.ingest.util\n\nimport java.lang.reflect.Field\nimport java.net.{InetSocketAddress, Sock"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V1IngestCodecs.scala",
"chars": 4359,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport cats.implicits.catsSyntaxEitherId\nimport io.circe.Encoder.encodeStri"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V1IngestSchemas.scala",
"chars": 1740,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport sttp.tapir.Schema\n\nimport com.thatdot.common.security.Secret\nimport "
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V1ToV2.scala",
"chars": 11998,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport com.thatdot.api.{v2 => api}\nimport com.thatdot.quine.app.model.inges"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V2IngestEntities.scala",
"chars": 20375,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport java.time.Instant\n\nimport scala.util.{Failure, Success, Try}\n\nimport"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V2IngestSources.scala",
"chars": 37123,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport java.nio.charset.Charset\nimport java.time.Instant\n\nimport io.circe.g"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/V2ToV1.scala",
"chars": 699,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport com.thatdot.quine.app.model.ingest2.{V2IngestEntities => V2}\nimport "
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/codec/FrameDecoder.scala",
"chars": 8982,
"preview": "package com.thatdot.quine.app.model.ingest2.codec\n\nimport java.io.StringReader\nimport java.nio.charset.{Charset, Standar"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/source/DecodedSource.scala",
"chars": 29565,
"preview": "package com.thatdot.quine.app.model.ingest2.source\n\nimport java.nio.charset.Charset\n\nimport scala.concurrent.duration.Du"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/source/FramedSource.scala",
"chars": 3909,
"preview": "package com.thatdot.quine.app.model.ingest2.source\n\nimport scala.util.Try\n\nimport org.apache.pekko.stream.scaladsl.{Flow"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/source/IngestBounds.scala",
"chars": 136,
"preview": "package com.thatdot.quine.app.model.ingest2.source\n\ncase class IngestBounds(startAtOffset: Long = 0L, ingestLimit: Optio"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/source/QuineIngestQuery.scala",
"chars": 5495,
"preview": "package com.thatdot.quine.app.model.ingest2.source\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/CsvFileSource.scala",
"chars": 2936,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.charset.{Charset, StandardCharsets}\n\nimport scala.u"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/FileSource.scala",
"chars": 9300,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.charset.{Charset, StandardCharsets}\n\nimport org.apa"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/FramedSourceProvider.scala",
"chars": 476,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport cats.data.ValidatedNel\n\nimport com.thatdot.quine.app.model.i"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/KafkaSource.scala",
"chars": 14689,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.util.UUID\n\nimport scala.concurrent.duration.{Duration, "
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/KinesisKclSrc.scala",
"chars": 21404,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.net.InetAddress\nimport java.nio.ByteBuffer\nimport java."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/KinesisSource.scala",
"chars": 8336,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.time.Instant\n\nimport scala.collection.Set\nimport scala."
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/NumberIteratorSource.scala",
"chars": 1621,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.ByteBuffer\n\nimport scala.util.{Success, Try}\n\nimpor"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/ReactiveSource.scala",
"chars": 2130,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport org.apache.pekko.NotUsed\nimport org.apache.pekko.actor.Actor"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/S3Source.scala",
"chars": 2135,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.charset.Charset\n\nimport org.apache.pekko.NotUsed\nim"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/ServerSentEventSource.scala",
"chars": 1960,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport org.apache.pekko.actor.ActorSystem\nimport org.apache.pekko.h"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/SqsSource.scala",
"chars": 4633,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport scala.jdk.CollectionConverters.MapHasAsScala\n\nimport org.apa"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/StandardInputSource.scala",
"chars": 1290,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.charset.Charset\n\nimport org.apache.pekko.NotUsed\nim"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/WebSocketClientSource.scala",
"chars": 4370,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport java.nio.charset.Charset\n\nimport scala.concurrent.duration.D"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/WebSocketFileUploadSource.scala",
"chars": 1614,
"preview": "package com.thatdot.quine.app.model.ingest2.sources\n\nimport scala.util.{Success, Try}\n\nimport org.apache.pekko.NotUsed\ni"
},
{
"path": "quine/src/main/scala/com/thatdot/quine/app/model/ingest2/sources/package.scala",
"chars": 1952,
"preview": "package com.thatdot.quine.app.model.ingest2\n\nimport java.nio.charset.{Charset, StandardCharsets}\n\nimport org.apache.pekk"
}
]
// ... and 862 more files (download for full content)
About this extraction
This page contains the full source code of the thatdot/quine GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 1062 files (6.6 MB), approximately 1.8M tokens, and a symbol index with 102 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.